libocxl
Loading...
Searching...
No Matches
mmio.c
Go to the documentation of this file.
1/*
2 * Copyright 2017 International Business Machines
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17// Needed for le32toh() and friends when building against glibc version < 2.20
18#define _BSD_SOURCE
19
20#include "libocxl_internal.h"
21#include "sys/mman.h"
22#include "errno.h"
23#include <sys/types.h>
24#include <sys/stat.h>
25#include <fcntl.h>
26#include <unistd.h>
27#include <string.h>
28#include <stdlib.h>
29
55static ocxl_err register_mmio(ocxl_afu *afu, void *addr, size_t size, ocxl_mmio_type type, ocxl_mmio_h *handle)
56{
57 int available_mmio = -1;
58
59 // Look for an available MMIO region that has been unmapped
60 for (uint16_t mmio = 0; mmio < afu->mmio_count; mmio++) {
61 if (!afu->mmios[mmio].start) {
62 available_mmio = mmio;
63 break;
64 }
65 }
66
67 if (available_mmio == -1) {
68 if (afu->mmio_count == afu->mmio_max_count) {
69 ocxl_err rc = grow_buffer(afu, (void **)&afu->mmios, &afu->mmio_max_count, sizeof(ocxl_mmio_area), INITIAL_MMIO_COUNT);
70 if (rc != OCXL_OK) {
71 errmsg(afu, rc, "Could not grow MMIO buffer");
72 return rc;
73 }
74 }
75
76 available_mmio = afu->mmio_count++;
77 }
78
79 afu->mmios[available_mmio].start = addr;
80 afu->mmios[available_mmio].length = size;
81 afu->mmios[available_mmio].type = type;
82 afu->mmios[available_mmio].afu = afu;
83
84 *handle = &afu->mmios[available_mmio];
85
86 TRACE(afu, "Mapped %ld bytes of %s MMIO at %p",
87 size, type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID", addr);
88
89 return OCXL_OK;
90}
91
100{
101 char path[PATH_MAX + 1];
102 int length = snprintf(path, sizeof(path), "%s/global_mmio_area", afu->sysfs_path);
103 if (length >= (int)sizeof(path)) {
105 errmsg(afu, rc, "global MMIO path truncated");
106 return rc;
107 }
108
109 int fd = open(path, O_RDWR | O_CLOEXEC);
110 if (fd < 0) {
112 errmsg(afu, rc, "Could not open global MMIO '%s': Error %d: %s", path, errno, strerror(errno));
113 return rc;
114 }
115
116 afu->global_mmio_fd = fd;
117
118 return OCXL_OK;
119}
120
144static ocxl_err global_mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset,
145 ocxl_mmio_h *region) // static function extraction hack
146{
147 if (afu->global_mmio.length == 0) {
149 errmsg(afu, rc, "Cannot map Global MMIO as there is 0 bytes allocated by the AFU");
150 return rc;
151 }
152
153 if (flags) {
155 errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
156 return rc;
157 }
158
159 void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->global_mmio_fd, offset);
160 if (addr == MAP_FAILED) {
162 errmsg(afu, rc, "Could not map global MMIO, %d: %s", errno, strerror(errno));
163 return rc;
164 }
165
166 ocxl_mmio_h mmio_region;
167 ocxl_err rc = register_mmio(afu, addr, size, OCXL_GLOBAL_MMIO, &mmio_region);
168 if (rc != OCXL_OK) {
169 errmsg(afu, rc, "Could not register global MMIO region");
170 munmap(addr, size);
171 return rc;
172 }
173
174 *region = mmio_region;
175
176 return OCXL_OK;
177}
178
202static ocxl_err mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
203{
204 if (flags) {
206 errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
207 return rc;
208 }
209
210 if (afu->fd < 0) {
212 errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been opened");
213 return rc;
214 }
215
216 if (!afu->attached) {
218 errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been attached");
219 return rc;
220 }
221
222 void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->fd, offset);
223 if (addr == MAP_FAILED) {
225 errmsg(afu, rc, "Could not map per-PASID MMIO: %d: %s", errno, strerror(errno));
226 return rc;
227 }
228
229 ocxl_mmio_h mmio_region;
230 ocxl_err rc = register_mmio(afu, addr, size, OCXL_PER_PASID_MMIO, &mmio_region);
231 if (rc != OCXL_OK) {
232 errmsg(afu, rc, "Could not register global MMIO region", afu->identifier.afu_name);
233 munmap(addr, size);
234 return rc;
235 }
236
237 *region = mmio_region;
238
239 return OCXL_OK;
240}
241
263ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags,
264 off_t offset, ocxl_mmio_h *region)
265{
267
268 if (size == 0) {
269 switch (type) {
271 size = afu->per_pasid_mmio.length;
272 break;
273 case OCXL_GLOBAL_MMIO:
274 size = afu->global_mmio.length;
275 break;
276 }
277
278 size -= offset;
279 }
280
281 switch (type) {
282 case OCXL_GLOBAL_MMIO:
283 if (offset + size > afu->global_mmio.length) {
284 rc = OCXL_NO_MEM;
285 errmsg(afu, rc, "Offset(%#x) + size(%#x) of global MMIO map request exceeds available size of %#x",
286 offset, size, afu->global_mmio.length);
287 return rc;
288 }
289 return global_mmio_map(afu, size, prot, flags, offset, region);
290
292 if (offset + size > afu->per_pasid_mmio.length) {
293 rc = OCXL_NO_MEM;
294 errmsg(afu, rc, "Offset(%#x) + size(%#x) of per-pasid MMIO map request exceeds available size of %#x",
295 offset, size, afu->global_mmio.length);
296 return rc;
297 }
298 return mmio_map(afu, size, prot, flags, offset, region);
299
300 default:
301 errmsg(afu, rc, "Unknown MMIO type %d", type);
302 return rc;
303 }
304}
305
325{
326 return ocxl_mmio_map_advanced(afu, type, 0, PROT_READ | PROT_WRITE, 0, 0, region);
327}
328
337{
338 if (!region->start) {
339 return;
340 }
341
342 munmap(region->start, region->length);
343 region->start = NULL;
344}
345
361{
362 switch (type) {
363 case OCXL_GLOBAL_MMIO:
364 return afu->global_mmio_fd;
365
367 return afu->fd;
368
369 default:
370 errmsg(afu, OCXL_INVALID_ARGS, "Unknown MMIO type %d", type);
371 return -1;
372 }
373}
374
384{
385 switch(type) {
386 case OCXL_GLOBAL_MMIO:
387 return afu->global_mmio.length;
388
390 return afu->per_pasid_mmio.length;
391
392 default:
393 errmsg(afu, OCXL_INVALID_ARGS, "Invalid MMIO area requested '%d'", type);
394 return 0;
395 }
396}
397
398
411ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
412{
413 if (!region->start) {
415 errmsg(region->afu, rc, "MMIO region has already been unmapped");
416 return rc;
417 }
418
419 *address = region->start;
420 *size = region->length;
421
422 return OCXL_OK;
423}
424
425
437inline static ocxl_err mmio_check(ocxl_mmio_h region, off_t offset, size_t size)
438{
439 if (!region) {
441 errmsg(NULL, rc, "MMIO region is invalid");
442 return rc;
443 }
444
445 if (!region->start) {
447 errmsg(region->afu, rc, "MMIO region has already been unmapped");
448 return rc;
449 }
450
451 if (offset >= (off_t)(region->length - (size - 1))) {
453 errmsg(region->afu, rc, "%s MMIO access of 0x%016lx exceeds limit of 0x%016lx",
454 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
455 offset, region->length);
456 return rc;
457 }
458
459 return OCXL_OK;
460}
461
480inline static ocxl_err mmio_read32_native(ocxl_mmio_h region, off_t offset, uint32_t *out)
481{
482 ocxl_err ret = mmio_check(region, offset, 4);
483 if (ret != OCXL_OK) {
484 return ret;
485 }
486
487 __sync_synchronize();
488 *out = *(volatile uint32_t *)(region->start + offset);
489 __sync_synchronize();
490
491 TRACE(region->afu, "%s MMIO Read32@0x%04lx=0x%08x",
492 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
493 offset, *out);
494
495 return OCXL_OK;
496}
497
516inline static ocxl_err mmio_read64_native(ocxl_mmio_h region, off_t offset, uint64_t *out)
517{
518 ocxl_err ret = mmio_check(region, offset, 8);
519 if (ret != OCXL_OK) {
520 return ret;
521 }
522
523 __sync_synchronize();
524 *out = *(volatile uint64_t *)(region->start + offset);
525 __sync_synchronize();
526
527 TRACE(region->afu, "%s MMIO Read64@0x%04lx=0x%016lx",
528 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
529 offset, *out);
530
531 return OCXL_OK;
532}
533
554inline static ocxl_err mmio_write32_native(ocxl_mmio_h region, off_t offset, uint32_t value)
555{
556 ocxl_err ret = mmio_check(region, offset, 4);
557 if (ret != OCXL_OK) {
558 return ret;
559 }
560
561 TRACE(region->afu, "%s MMIO Write32@0x%04lx=0x%08x",
562 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
563 offset, value);
564
565 volatile uint32_t *addr = (uint32_t *)(region->start + offset);
566
567 __sync_synchronize();
568 *addr = value;
569 __sync_synchronize();
570
571 return OCXL_OK;
572}
573
592inline static ocxl_err mmio_write64_native(ocxl_mmio_h region, off_t offset, uint64_t value)
593{
594 ocxl_err ret = mmio_check(region, offset, 8);
595 if (ret != OCXL_OK) {
596 return ret;
597 }
598
599 TRACE(region->afu, "%s MMIO Write64@0x%04lx=0x%016lx",
600 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
601 offset, value);
602
603 volatile uint64_t *addr = (uint64_t *)(region->start + offset);
604
605 __sync_synchronize();
606 *addr = value;
607 __sync_synchronize();
608
609 return OCXL_OK;
610}
611
630ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
631{
632 uint32_t val;
633 ocxl_err ret = mmio_read32_native(mmio, offset, &val);
634
635 if (UNLIKELY(ret != OCXL_OK)) {
636 return ret;
637 }
638
639 switch (endian) {
641 *out = be32toh(val);
642 break;
643
645 *out = le32toh(val);
646 break;
647 default:
648 *out = val;
649 break;
650 }
651
652 return OCXL_OK;
653}
654
674ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
675{
676 uint64_t val;
677 ocxl_err ret = mmio_read64_native(mmio, offset, &val);
678
679 if (UNLIKELY(ret != OCXL_OK)) {
680 return ret;
681 }
682
683 switch (endian) {
685 *out = be64toh(val);
686 break;
687
689 *out = le64toh(val);
690 break;
691 default:
692 *out = val;
693 break;
694 }
695
696 return OCXL_OK;
697}
698
718ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
719{
720 switch (endian) {
722 value = htobe32(value);
723 break;
724
726 value = htole32(value);
727 break;
728 default:
729 break;
730 }
731
732 return mmio_write32_native(mmio, offset, value);
733}
734
753ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
754{
755 switch (endian) {
757 value = htobe64(value);
758 break;
759
761 value = htole64(value);
762 break;
763 default:
764 break;
765 }
766
767 return mmio_write64_native(mmio, offset, value);
768}
769
770
size_t ocxl_mmio_size(ocxl_afu_h afu, ocxl_mmio_type type)
Get the size of an MMIO region for an AFU.
Definition mmio.c:383
ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
Convert endianness and write a 64-bit value to an AFU's MMIO region.
Definition mmio.c:753
int ocxl_mmio_get_fd(ocxl_afu_h afu, ocxl_mmio_type type)
Get a file descriptor for an MMIO area of an AFU.
Definition mmio.c:360
ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition mmio.c:263
ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
Read a 64-bit value from an AFU's MMIO region & convert endianness.
Definition mmio.c:674
void ocxl_mmio_unmap(ocxl_mmio_h region)
Unmap an MMIO region from an AFU.
Definition mmio.c:336
ocxl_err ocxl_mmio_map(ocxl_afu_h afu, ocxl_mmio_type type, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition mmio.c:324
ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
Convert endianness and write a 32-bit value to an AFU's MMIO region.
Definition mmio.c:718
ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
Read a 32-bit value from an AFU's MMIO region & convert endianness.
Definition mmio.c:630
ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
Get the address & size of a mapped MMIO region.
Definition mmio.c:411
ocxl_err global_mmio_open(ocxl_afu *afu)
Open the global MMIO descriptor on an AFU.
Definition mmio.c:99
ocxl_err
Potential return values from ocxl_* functions.
Definition libocxl.h:92
@ OCXL_NO_DEV
The OpenCAPI device is not available.
Definition libocxl.h:95
@ OCXL_INVALID_ARGS
One or more arguments are invalid.
Definition libocxl.h:102
@ OCXL_OUT_OF_BOUNDS
The action requested falls outside the permitted area.
Definition libocxl.h:100
@ OCXL_OK
The call succeeded.
Definition libocxl.h:93
@ OCXL_NO_MEM
An out of memory error occurred.
Definition libocxl.h:94
@ OCXL_NO_CONTEXT
The call requires an open context on the AFU.
Definition libocxl.h:96
ocxl_mmio_type
Defines the type of an MMIO area.
Definition libocxl.h:56
@ OCXL_GLOBAL_MMIO
Definition libocxl.h:57
@ OCXL_PER_PASID_MMIO
Definition libocxl.h:58
struct ocxl_afu * ocxl_afu_h
A handle for an AFU.
Definition libocxl.h:74
struct ocxl_mmio_area * ocxl_mmio_h
A handle for an MMIO region on an AFU.
Definition libocxl.h:86
ocxl_endian
Defines the endianness of an AFU MMIO area.
Definition libocxl.h:47
@ OCXL_MMIO_LITTLE_ENDIAN
AFU data is little-endian.
Definition libocxl.h:49
@ OCXL_MMIO_BIG_ENDIAN
AFU data is big-endian.
Definition libocxl.h:48