Skip to content

ENH: Add support for PCPUs #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
AlexCK-STFC wants to merge 3 commits into from
Closed

ENH: Add support for PCPUs #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f130494
ENH: Add support for PCPUs
AlexCK-STFC Oct 1, 2025
c0920b8
MAINT: Bump minor
AlexCK-STFC Oct 1, 2025
fae8fae
TST: Add test coverage for logic in hypervisor prop mapping
AlexCK-STFC Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions docs/user_docs/query_docs/HYPERVISORS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,24 @@ from openstackquery import HypervisorQuery

`Hypervisors` have the following properties:

| Return Type | Property Name(s) (case-insensitive) | Description |
|-------------|------------------------------------------------------------------|-----------------------------------------------------------------|
| `int` | "disk_gb_avail", "disk_avail", "local_disk_free", "free_disk_gb" | The local disk space remaining on this hypervisor(in GiB) |
| `int` | "disk_gb_used", "disk_used", "local_disk_used", "local_gb_used" | The local disk space allocated on this hypervisor(in GiB) |
| `int` | "disk_gb_size", "disk", "local_disk", "local_gb" | The total amount of local disk space(in GiB) |
| `string` | "id", "uuid", "host_id" | ID of the Hypervisor |
| `string` | "ip", "host_ip" | The IP address of the hypervisor’s host |
| `int` | "memory_mb_avail", "memory_avail", "memory_free", "free_ram_mb" | The free RAM on this hypervisor(in MiB). |
| `int` | "memory_used", "memory_mb_used" | RAM currently being used on this hypervisor(in MiB). |
| `int` | "memory_mb_size", "memory_size", "memory_mb", "memory", "ram" | The total amount of ram(in MiB) |
| `string` | "name", "host_name" | Hypervisor Hostname |
| `string` | "state" | The state of the hypervisor. One of up or down. |
| `string` | "status" | The status of the hypervisor. One of enabled or disabled. |
| `int` | "vcpus" | The number of vCPUs on this hypervisor. |
| `int` | "vcpus_free" "vcpus_avail" | The number of vCPUs on this hypervisor. |
| `int` | "vcpus_used", "vcpus_in_use" | The number of vCPUs currently being used on this hypervisor. |
| `string` | "disabled_reason" | Comment of why the hypervisor is disabled, None if not disabled |
| `float` | "uptime" | The total uptime in days of the hypervisor |
| Return Type | Property Name(s) (case-insensitive) | Description |
|-------------|----------------------------------------------------------------------------------------|--------------------------------------------------------------------|
| `int` | "disk_gb_avail", "disk_avail", "local_disk_free", "free_disk_gb" | The local disk space remaining on this hypervisor(in GiB) |
| `int` | "disk_gb_used", "disk_used", "local_disk_used", "local_gb_used" | The local disk space allocated on this hypervisor(in GiB) |
| `int` | "disk_gb_size", "disk", "local_disk", "local_gb" | The total amount of local disk space(in GiB) |
| `string` | "id", "uuid", "host_id" | ID of the Hypervisor |
| `string` | "ip", "host_ip" | The IP address of the hypervisor’s host |
| `int` | "memory_mb_avail", "memory_avail", "memory_free", "free_ram_mb" | The free RAM on this hypervisor(in MiB). |
| `int` | "memory_used", "memory_mb_used" | RAM currently being used on this hypervisor(in MiB). |
| `int` | "memory_mb_size", "memory_size", "memory_mb", "memory", "ram" | The total amount of ram(in MiB) |
| `string` | "name", "host_name" | Hypervisor Hostname |
| `string` | "state" | The state of the hypervisor. One of up or down. |
| `string` | "status" | The status of the hypervisor. One of enabled or disabled. |
| `int` | "vcpus", "cpus", "pcpus" | The number of vCPUs/pCPUs on this hypervisor. |
| `int` | "vcpus_free", "vcpus_avail", "cpus_avail", "cpus_free", "pcpus_avail", "pcpus_free" | The number of vCPUs/pCPUs on this hypervisor. |
| `int` | "vcpus_used", "vcpus_in_use", "cpus_used", "cpus_in_use", "pcpus_used", "pcpus_in_use" | The number of vCPUs/pCPUs currently being used on this hypervisor. |
| `string` | "disabled_reason" | Comment of why the hypervisor is disabled, None if not disabled |
| `float` | "uptime" | The total uptime in days of the hypervisor |

Any of these properties can be used for any of the API methods that takes a property - like `select`, `where`, `sort_by` etc

Expand Down
29 changes: 22 additions & 7 deletions openstackquery/enums/props/hypervisor_properties.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,23 @@ def _get_aliases() -> Dict:
HypervisorProperties.HYPERVISOR_STATUS: ["status"],
HypervisorProperties.HYPERVISOR_DISABLED_REASON: ["disabled_reason"],
HypervisorProperties.HYPERVISOR_UPTIME_DAYS: ["uptime"],
HypervisorProperties.VCPUS: ["vcpus"],
HypervisorProperties.VCPUS_USED: ["vcpus_used", "vcpus_in_use"],
HypervisorProperties.VCPUS_AVAIL: ["vcpus_avail", "vcpus_free"],
HypervisorProperties.VCPUS: ["vcpus", "cpus", "pcpus"],
HypervisorProperties.VCPUS_USED: [
"vcpus_used",
"vcpus_in_use",
"cpus_used",
"cpus_in_use",
"pcpus_used",
"pcpus_in_use",
],
HypervisorProperties.VCPUS_AVAIL: [
"vcpus_avail",
"vcpus_free",
"cpus_avail",
"cpus_free",
"pcpus_avail",
"pcpus_free",
],
HypervisorProperties.MEMORY_MB_SIZE: [
"memory_mb_size",
"memory_size",
Expand Down Expand Up @@ -92,7 +106,6 @@ def get_prop_mapping(prop) -> Optional[PropFunc]:
HypervisorProperties.HYPERVISOR_ID: lambda a: a.hv["id"],
HypervisorProperties.HYPERVISOR_IP: lambda a: a.hv["host_ip"],
HypervisorProperties.HYPERVISOR_NAME: lambda a: a.hv["name"],
# HypervisorProperties.HYPERVISOR_SERVER_COUNT: lambda a: a["runnning_vms"],
HypervisorProperties.HYPERVISOR_STATE: lambda a: a.hv["state"],
HypervisorProperties.HYPERVISOR_STATUS: lambda a: a.hv["status"],
HypervisorProperties.HYPERVISOR_DISABLED_REASON: lambda a: a.hv["service"][
Expand All @@ -101,13 +114,15 @@ def get_prop_mapping(prop) -> Optional[PropFunc]:
HypervisorProperties.HYPERVISOR_UPTIME_DAYS: lambda a: TimeUtils.extract_uptime(
a.hv["uptime"]
),
HypervisorProperties.VCPUS: lambda a: a.usage.vcpus,
HypervisorProperties.VCPUS_AVAIL: lambda a: a.usage.vcpus_avail,
HypervisorProperties.VCPUS: lambda a: a.usage.vcpus + a.usage.pcpus,
HypervisorProperties.VCPUS_AVAIL: lambda a: a.usage.vcpus_avail
+ a.usage.pcpus_avail,
HypervisorProperties.VCPUS_USED: lambda a: a.usage.vcpus_used
+ a.usage.pcpus_used,
Comment on lines +117 to +121
Copy link
Collaborator

@gmatthews20 gmatthews20 Oct 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know if we should be combining PCPU and VCPU like this since they are two different resources in openstack, its only our specific use case where we only use PCPUs that we run into this issue. I'd suggest return separate fields for PCPU and VCPU and let the users of the library decided how to handle them

HypervisorProperties.MEMORY_MB_SIZE: lambda a: a.usage.memory_mb_size,
HypervisorProperties.MEMORY_MB_AVAIL: lambda a: a.usage.memory_mb_avail,
HypervisorProperties.DISK_GB_SIZE: lambda a: a.usage.disk_gb_size,
HypervisorProperties.DISK_GB_AVAIL: lambda a: a.usage.disk_gb_avail,
HypervisorProperties.VCPUS_USED: lambda a: a.usage.vcpus_used,
HypervisorProperties.MEMORY_MB_USED: lambda a: a.usage.memory_mb_used,
HypervisorProperties.DISK_GB_USED: lambda a: a.usage.disk_gb_used,
}
Expand Down
10 changes: 7 additions & 3 deletions openstackquery/runners/hypervisor_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import logging
from typing import List, Optional, Dict
from typing import Dict, List, Optional

from openstack import exceptions, utils
from openstack.compute.v2.hypervisor import Hypervisor as OpenstackHypervisor
Expand All @@ -9,7 +9,6 @@
from openstackquery.openstack_connection import OpenstackConnection
from openstackquery.runners.runner_utils import RunnerUtils
from openstackquery.runners.runner_wrapper import RunnerWrapper

from openstackquery.structs.hypervisor import Hypervisor
from openstackquery.structs.resource_provider_usage import ResourceProviderUsage

Expand Down Expand Up @@ -69,6 +68,8 @@ def _convert_to_custom_obj(
memory_mb_used = usage.get("MEMORY_MB", 0)
disk_gb_used = usage.get("DISK_GB", 0)

pcpus_used = usage.get("PCPU", 0)

return ResourceProviderUsage(
# workaround for hvs not containing VCPU/Memory/Disk resource provider info - set to 0
vcpus_used=vcpus_used,
Expand All @@ -80,6 +81,9 @@ def _convert_to_custom_obj(
vcpus=avail["VCPU"] + vcpus_used,
memory_mb_size=avail["MEMORY_MB"] + memory_mb_used,
disk_gb_size=avail["DISK_GB"] + disk_gb_used,
pcpus_avail=avail["PCPU"],
pcpus_used=pcpus_used,
pcpus=pcpus_used + avail["PCPU"],
)

@staticmethod
Expand All @@ -94,7 +98,7 @@ def _get_availability_info(
:return: A dictionary with the summed availability stats using the class name as a key
"""
summed_classes = {}
for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB"]:
for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB", "PCPU"]:
placement_inventories = conn.placement.resource_provider_inventories(
resource_provider_obj, resource_class=resource_class
)
Expand Down
4 changes: 4 additions & 0 deletions openstackquery/structs/resource_provider_usage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,7 @@ class ResourceProviderUsage:
vcpus_used: int
memory_mb_used: int
disk_gb_used: int

pcpus: int
pcpus_used: int
pcpus_avail: int
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

setup(
name="openstackquery",
version="2.0.0",
version="2.1.0",
author="STFC Cloud Team",
author_email="<[email protected]>",
description=DESCRIPTION,
Expand Down
148 changes: 144 additions & 4 deletions tests/enums/props/test_hypervisor_properties.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
from dataclasses import dataclass
from unittest.mock import patch

import pytest

from openstackquery.enums.props.hypervisor_properties import HypervisorProperties
from openstackquery.exceptions.query_property_mapping_error import (
QueryPropertyMappingError,
)

from tests.mocks.mocked_props import MockProperties


Expand All @@ -28,7 +29,17 @@
["hypervisor_disabled_reason", "disabled_reason"],
),
(HypervisorProperties.HYPERVISOR_UPTIME_DAYS, ["hypervisor_uptime_days"]),
(HypervisorProperties.VCPUS_AVAIL, ["vcpus_avail"]),
(
HypervisorProperties.VCPUS_AVAIL,
[
"vcpus_avail",
"vcpus_free",
"cpus_avail",
"cpus_free",
"pcpus_avail",
"pcpus_free",
],
),
(
HypervisorProperties.MEMORY_MB_AVAIL,
["memory_mb_avail", "memory_avail", "memory_free", "free_ram_mb"],
Expand All @@ -37,7 +48,17 @@
HypervisorProperties.DISK_GB_AVAIL,
["disk_gb_avail", "disk_avail", "local_disk_free", "free_disk_gb"],
),
(HypervisorProperties.VCPUS_USED, ["vcpus_used", "vcpus_in_use"]),
(
HypervisorProperties.VCPUS_USED,
[
"vcpus_used",
"vcpus_in_use",
"cpus_used",
"cpus_in_use",
"pcpus_used",
"pcpus_in_use",
],
),
(HypervisorProperties.MEMORY_MB_USED, ["memory_mb_used", "memory_used"]),
(
HypervisorProperties.DISK_GB_USED,
Expand All @@ -51,7 +72,7 @@
HypervisorProperties.MEMORY_MB_SIZE,
["memory_mb_size", "memory_size", "memory_mb", "memory", "ram"],
),
(HypervisorProperties.VCPUS, ["vcpus"]),
(HypervisorProperties.VCPUS, ["vcpus", "cpus", "pcpus"]),
],
)
def test_property_serialization(expected_prop, test_values, property_variant_generator):
Expand Down Expand Up @@ -86,3 +107,122 @@ def test_get_marker_prop_func(mock_get_prop_mapping):
val = HypervisorProperties.get_marker_prop_func()
mock_get_prop_mapping.assert_called_once_with(HypervisorProperties.HYPERVISOR_ID)
assert val == mock_get_prop_mapping.return_value


# pylint: disable=too-many-instance-attributes
@dataclass
class MockUsage:
vcpus: int = 0
pcpus: int = 0
vcpus_used: int = 0
pcpus_used: int = 0
vcpus_avail: int = 0
pcpus_avail: int = 0
memory_mb_size: int = 0
memory_mb_used: int = 0
memory_mb_avail: int = 0
disk_gb_size: int = 0
disk_gb_used: int = 0
disk_gb_avail: int = 0


@dataclass
class MockHypervisor:
hv: dict
usage: MockUsage


@pytest.mark.parametrize(
"prop, mock_hv, mock_usage, expected",
[
(HypervisorProperties.HYPERVISOR_ID, {"id": "abc123"}, None, "abc123"),
(HypervisorProperties.HYPERVISOR_IP, {"host_ip": "10.0.0.1"}, None, "10.0.0.1"),
(HypervisorProperties.HYPERVISOR_NAME, {"name": "hyp1"}, None, "hyp1"),
(HypervisorProperties.HYPERVISOR_STATE, {"state": "up"}, None, "up"),
(
HypervisorProperties.HYPERVISOR_STATUS,
{"status": "enabled"},
None,
"enabled",
),
(
HypervisorProperties.HYPERVISOR_DISABLED_REASON,
{"service": {"disabled_reason": "maintenance"}},
None,
"maintenance",
),
(
HypervisorProperties.VCPUS,
{},
MockUsage(vcpus=4, pcpus=2),
6,
),
(
HypervisorProperties.VCPUS_USED,
{},
MockUsage(vcpus_used=1, pcpus_used=2),
3,
),
(
HypervisorProperties.VCPUS_AVAIL,
{},
MockUsage(vcpus_avail=3, pcpus_avail=1),
4,
),
(
HypervisorProperties.MEMORY_MB_SIZE,
{},
MockUsage(memory_mb_size=8192),
8192,
),
(
HypervisorProperties.MEMORY_MB_USED,
{},
MockUsage(memory_mb_used=4096),
4096,
),
(
HypervisorProperties.MEMORY_MB_AVAIL,
{},
MockUsage(memory_mb_avail=2048),
2048,
),
(
HypervisorProperties.DISK_GB_SIZE,
{},
MockUsage(disk_gb_size=500),
500,
),
(
HypervisorProperties.DISK_GB_USED,
{},
MockUsage(disk_gb_used=200),
200,
),
(
HypervisorProperties.DISK_GB_AVAIL,
{},
MockUsage(disk_gb_avail=300),
300,
),
],
)
def test_hypervisor_property_mappings(prop, mock_hv, mock_usage, expected):
"""Test that each HypervisorProperties mapping correctly extracts the expected data."""
hv_obj = MockHypervisor(mock_hv, mock_usage)
func = HypervisorProperties.get_prop_mapping(prop)
assert func(hv_obj) == expected


@patch("openstackquery.enums.props.hypervisor_properties.TimeUtils.extract_uptime")
def test_hypervisor_uptime_days_mapping(mock_extract):
mock_extract.return_value = 5
hv_obj = MockHypervisor({"uptime": "fake-uptime-string"}, None)

func = HypervisorProperties.get_prop_mapping(
HypervisorProperties.HYPERVISOR_UPTIME_DAYS
)
result = func(hv_obj)

mock_extract.assert_called_once_with("fake-uptime-string")
assert result == 5
Loading