mawei/sunnypilot
1
0
Fork 0
mirror of https://github.com/sunnypilot/sunnypilot.git synced 2026-02-18 22:23:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

use build_metadata everywhere we used to use get_version, get_commit, etc (#31941)

Browse Source 
* use build_metadata

* fix normailzed

* also normalized

* and here

* fix diff

* and that one

* cleanup
This commit is contained in:
Justin Newberry
2024-03-21 13:15:29 -04:00
committed by GitHub
parent 806f743e12
commit effee900c4
10 changed files with 118 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
selfdrive/athena/athenad.py
View File

@@ -32,13 +32,12 @@ from cereal import log
from cereal.services import SERVICE_LIST
from openpilot.common.api import Api
from openpilot.common.file_helpers import CallbackReader
from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch
from openpilot.common.params import Params
from openpilot.common.realtime import set_core_affinity
from openpilot.system.hardware import HARDWARE, PC
from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
from openpilot.common.swaglog import cloudlog
from openpilot.system.version import get_version
from openpilot.system.version import get_build_metadata
from openpilot.system.hardware.hw import Paths
@@ -320,11 +319,12 @@ def getMessage(service: str, timeout: int = 1000) -> dict:
@dispatcher.add_method
def getVersion() -> dict[str, str]:
build_metadata = get_build_metadata()
return {
"version": get_version(),
"remote": get_normalized_origin(),
"branch": get_short_branch(),
"commit": get_commit(),
"version": build_metadata.openpilot.version,
"remote": build_metadata.openpilot.git_normalized_origin,
"branch": build_metadata.channel,
"commit": build_metadata.openpilot.git_commit,
}

15
selfdrive/athena/manage_athenad.py
View File

@@ -7,8 +7,7 @@ from openpilot.common.params import Params
from openpilot.selfdrive.manager.process import launcher
from openpilot.common.swaglog import cloudlog
from openpilot.system.hardware import HARDWARE
from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch
from openpilot.system.version import get_version, is_dirty
from openpilot.system.version import get_build_metadata
ATHENA_MGR_PID_PARAM = "AthenadPid"
@@ -16,12 +15,14 @@ ATHENA_MGR_PID_PARAM = "AthenadPid"
def main():
params = Params()
dongle_id = params.get("DongleId").decode('utf-8')
build_metadata = get_build_metadata()
cloudlog.bind_global(dongle_id=dongle_id,
version=get_version(),
origin=get_normalized_origin(),
branch=get_short_branch(),
commit=get_commit(),
dirty=is_dirty(),
version=build_metadata.openpilot.version,
origin=build_metadata.openpilot.git_normalized_origin,
branch=build_metadata.channel,
commit=build_metadata.openpilot.git_commit,
dirty=build_metadata.openpilot.is_dirty,
device=HARDWARE.get_device_type())
try:

5
selfdrive/car/car_helpers.py
View File

@@ -4,7 +4,6 @@ from collections.abc import Callable
from cereal import car
from openpilot.common.params import Params
from openpilot.system.version import is_comma_remote, is_tested_branch
from openpilot.selfdrive.car.interfaces import get_interface_attr
from openpilot.selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars
from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN
@@ -13,6 +12,7 @@ from openpilot.selfdrive.car.mock.values import CAR as MOCK
from openpilot.common.swaglog import cloudlog
import cereal.messaging as messaging
from openpilot.selfdrive.car import gen_empty_fingerprint
from openpilot.system.version import get_build_metadata
FRAME_FINGERPRINT = 100 # 1s
@@ -20,7 +20,8 @@ EventName = car.CarEvent.EventName
def get_startup_event(car_recognized, controller_available, fw_seen):
if is_comma_remote() and is_tested_branch():
build_metadata = get_build_metadata()
if build_metadata.openpilot.comma_remote and build_metadata.tested_channel:
event = EventName.startup
else:
event = EventName.startupMaster

5
selfdrive/manager/build.py
View File

@@ -9,7 +9,7 @@ from openpilot.common.spinner import Spinner
from openpilot.common.text_window import TextWindow
from openpilot.system.hardware import AGNOS
from openpilot.common.swaglog import cloudlog, add_file_handler
from openpilot.system.version import is_dirty
from openpilot.system.version import get_build_metadata
MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
@@ -86,4 +86,5 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
if __name__ == "__main__":
spinner = Spinner()
spinner.update_progress(0, 100)
build(spinner, is_dirty(), minimal = AGNOS)
build_metadata = get_build_metadata()
build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS)

41
selfdrive/manager/manager.py
View File

@@ -16,21 +16,20 @@ from openpilot.selfdrive.manager.process import ensure_running
from openpilot.selfdrive.manager.process_config import managed_processes
from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID
from openpilot.common.swaglog import cloudlog, add_file_handler
from openpilot.common.git import get_commit, get_origin, get_short_branch, get_commit_date
from openpilot.system.version import is_dirty, get_version, \
get_normalized_origin, terms_version, training_version, \
is_tested_branch, is_release_branch
from openpilot.system.version import get_build_metadata, terms_version, training_version
def manager_init() -> None:
save_bootlog()
build_metadata = get_build_metadata()
params = Params()
params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
if is_release_branch():
if build_metadata.release_channel:
params.clear_all(ParamKeyType.DEVELOPMENT_ONLY)
default_params: list[tuple[str, str | bytes]] = [
@@ -62,15 +61,15 @@ def manager_init() -> None:
print("WARNING: failed to make /dev/shm")
# set version params
params.put("Version", get_version())
params.put("Version", build_metadata.openpilot.version)
params.put("TermsVersion", terms_version)
params.put("TrainingVersion", training_version)
params.put("GitCommit", get_commit())
params.put("GitCommitDate", get_commit_date())
params.put("GitBranch", get_short_branch())
params.put("GitRemote", get_origin())
params.put_bool("IsTestedBranch", is_tested_branch())
params.put_bool("IsReleaseBranch", is_release_branch())
params.put("GitCommit", build_metadata.openpilot.git_commit)
params.put("GitCommitDate", build_metadata.openpilot.git_commit_date)
params.put("GitBranch", build_metadata.channel)
params.put("GitRemote", build_metadata.openpilot.git_origin)
params.put_bool("IsTestedBranch", build_metadata.tested_channel)
params.put_bool("IsReleaseBranch", build_metadata.release_channel)
# set dongle id
reg_res = register(show_spinner=True)
@@ -80,21 +79,21 @@ def manager_init() -> None:
serial = params.get("HardwareSerial")
raise Exception(f"Registration failed for device {serial}")
os.environ['DONGLE_ID'] = dongle_id # Needed for swaglog
os.environ['GIT_ORIGIN'] = get_normalized_origin() # Needed for swaglog
os.environ['GIT_BRANCH'] = get_short_branch() # Needed for swaglog
os.environ['GIT_COMMIT'] = get_commit() # Needed for swaglog
os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog
os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog
os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog
if not is_dirty():
if not build_metadata.openpilot.is_dirty:
os.environ['CLEAN'] = '1'
# init logging
sentry.init(sentry.SentryProject.SELFDRIVE)
cloudlog.bind_global(dongle_id=dongle_id,
version=get_version(),
origin=get_normalized_origin(),
branch=get_short_branch(),
commit=get_commit(),
dirty=is_dirty(),
version=build_metadata.openpilot.version,
origin=build_metadata.openpilot.git_normalized_origin,
branch=build_metadata.channel,
commit=build_metadata.openpilot.git_commit,
dirty=build_metadata.openpilot.is_dirty,
device=HARDWARE.get_device_type())
# preimport all processes

18
selfdrive/sentry.py
View File

@@ -6,9 +6,8 @@ from sentry_sdk.integrations.threading import ThreadingIntegration
from openpilot.common.params import Params
from openpilot.selfdrive.athena.registration import is_registered_device
from openpilot.system.hardware import HARDWARE, PC
from openpilot.common.git import get_commit, get_branch, get_origin
from openpilot.common.swaglog import cloudlog
from openpilot.system.version import get_version, is_comma_remote, is_dirty, is_tested_branch
from openpilot.system.version import get_build_metadata, get_version
class SentryProject(Enum):
@@ -43,12 +42,13 @@ def set_tag(key: str, value: str) -> None:
def init(project: SentryProject) -> bool:
build_metadata = get_build_metadata()
# forks like to mess with this, so double check
comma_remote = is_comma_remote() and "commaai" in get_origin()
comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin
if not comma_remote or not is_registered_device() or PC:
return False
env = "release" if is_tested_branch() else "master"
env = "release" if build_metadata.tested_channel else "master"
dongle_id = Params().get("DongleId", encoding='utf-8')
integrations = []
@@ -63,11 +63,13 @@ def init(project: SentryProject) -> bool:
max_value_length=8192,
environment=env)
build_metadata = get_build_metadata()
sentry_sdk.set_user({"id": dongle_id})
sentry_sdk.set_tag("dirty", is_dirty())
sentry_sdk.set_tag("origin", get_origin())
sentry_sdk.set_tag("branch", get_branch())
sentry_sdk.set_tag("commit", get_commit())
sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty)
sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin)
sentry_sdk.set_tag("branch", build_metadata.channel)
sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit)
sentry_sdk.set_tag("device", HARDWARE.get_device_type())
if project == SentryProject.SELFDRIVE:

12
selfdrive/statsd.py
View File

@@ -13,7 +13,7 @@ from openpilot.system.hardware.hw import Paths
from openpilot.common.swaglog import cloudlog
from openpilot.system.hardware import HARDWARE
from openpilot.common.file_helpers import atomic_write_in_dir
from openpilot.system.version import get_normalized_origin, get_short_branch, get_short_version, is_dirty
from openpilot.system.version import get_build_metadata
from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
@@ -86,13 +86,15 @@ def main() -> NoReturn:
# initialize stats directory
Path(STATS_DIR).mkdir(parents=True, exist_ok=True)
build_metadata = get_build_metadata()
# initialize tags
tags = {
'started': False,
'version': get_short_version(),
'branch': get_short_branch(),
'dirty': is_dirty(),
'origin': get_normalized_origin(),
'version': build_metadata.openpilot.version,
'branch': build_metadata.channel,
'dirty': build_metadata.openpilot.is_dirty,
'origin': build_metadata.openpilot.git_normalized_origin,
'deviceType': HARDWARE.get_device_type(),
}

6
selfdrive/tombstoned.py
View File

@@ -11,8 +11,8 @@ from typing import NoReturn
import openpilot.selfdrive.sentry as sentry
from openpilot.system.hardware.hw import Paths
from openpilot.common.git import get_commit
from openpilot.common.swaglog import cloudlog
from openpilot.system.version import get_build_metadata
MAX_SIZE = 1_000_000 * 100 # allow up to 100M
MAX_TOMBSTONE_FN_LEN = 62 # 85 - 23 ("<dongle id>/crash/")
@@ -124,7 +124,9 @@ def report_tombstone_apport(fn):
clean_path = path.replace('/', '_')
date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S")
new_fn = f"{date}_{(get_commit() or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN]
build_metadata = get_build_metadata()
new_fn = f"{date}_{(build_metadata.openpilot.git_commit or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN]
crashlog_dir = os.path.join(Paths.log_root(), "crash")
os.makedirs(crashlog_dir, exist_ok=True)

5
selfdrive/updated/updated.py
View File

@@ -19,7 +19,7 @@ from openpilot.common.time import system_time_valid
from openpilot.system.hardware import AGNOS, HARDWARE
from openpilot.common.swaglog import cloudlog
from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
from openpilot.system.version import is_tested_branch
from openpilot.system.version import get_build_metadata
LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")
@@ -325,8 +325,9 @@ class Updater:
now = datetime.datetime.utcnow()
dt = now - last_update
build_metadata = get_build_metadata()
if failed_count > 15 and exception is not None and self.has_internet:
if is_tested_branch():
if build_metadata.tested_channel:
extra_text = "Ensure the software is correctly installed. Uninstall and re-install if this error persists."
else:
extra_text = exception

88
system/version.py
View File

@@ -9,7 +9,7 @@ import subprocess
from openpilot.common.basedir import BASEDIR
from openpilot.common.swaglog import cloudlog
from openpilot.common.utils import cache
from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_normalized_origin, get_commit_date
from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_commit_date
RELEASE_BRANCHES = ['release3-staging', 'release3', 'nightly']
@@ -32,33 +32,15 @@ def get_release_notes(path: str = BASEDIR) -> str:
return f.read().split('\n\n', 1)[0]
@cache
def get_short_version() -> str:
return get_version().split('-')[0]
@cache
def is_prebuilt(path: str = BASEDIR) -> bool:
return os.path.exists(os.path.join(path, 'prebuilt'))
@cache
def is_comma_remote() -> bool:
# note to fork maintainers, this is used for release metrics. please do not
# touch this to get rid of the orange startup alert. there's better ways to do that
return get_normalized_origin() == "github.com/commaai/openpilot"
@cache
def is_tested_branch() -> bool:
return get_short_branch() in TESTED_BRANCHES
@cache
def is_release_branch() -> bool:
return get_short_branch() in RELEASE_BRANCHES
@cache
def is_dirty(cwd: str = BASEDIR) -> bool:
origin = get_origin(cwd)
branch = get_branch(cwd)
origin = get_origin()
branch = get_branch()
if not origin or not branch:
return True
@@ -85,6 +67,27 @@ class OpenpilotMetadata:
version: str
release_notes: str
git_commit: str
git_origin: str
git_commit_date: str
is_dirty: bool # whether there are local changes
@property
def short_version(self) -> str:
return self.version.split('-')[0]
@property
def comma_remote(self) -> bool:
# note to fork maintainers, this is used for release metrics. please do not
# touch this to get rid of the orange startup alert. there's better ways to do that
return self.git_normalized_origin == "github.com/commaai/openpilot"
@property
def git_normalized_origin(self) -> str:
return self.git_origin \
.replace("git@", "", 1) \
.replace(".git", "", 1) \
.replace("https://", "", 1) \
.replace(":", "/", 1)
@dataclass(frozen=True)
@@ -92,9 +95,17 @@ class BuildMetadata:
channel: str
openpilot: OpenpilotMetadata
@property
def tested_channel(self) -> bool:
return self.channel in TESTED_BRANCHES
@property
def release_channel(self) -> bool:
return self.channel in RELEASE_BRANCHES
def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None:
def get_build_metadata(path: str = BASEDIR) -> BuildMetadata:
build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME
if build_metadata_path.exists():
@@ -105,14 +116,31 @@ def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None:
version = openpilot_metadata.get("version", "unknown")
release_notes = openpilot_metadata.get("release_notes", "unknown")
git_commit = openpilot_metadata.get("git_commit", "unknown")
return BuildMetadata(channel, OpenpilotMetadata(version, release_notes, git_commit))
git_origin = openpilot_metadata.get("git_origin", "unknown")
git_commit_date = openpilot_metadata.get("git_commit_date", "unknown")
return BuildMetadata(channel,
OpenpilotMetadata(
version=version,
release_notes=release_notes,
git_commit=git_commit,
git_origin=git_origin,
git_commit_date=git_commit_date,
is_dirty=False))
git_folder = pathlib.Path(path) / ".git"
if git_folder.exists():
return BuildMetadata(get_short_branch(path), OpenpilotMetadata(get_version(path), get_release_notes(path), get_commit(path)))
return BuildMetadata(get_short_branch(path),
OpenpilotMetadata(
version=get_version(path),
release_notes=get_release_notes(path),
git_commit=get_commit(path),
git_origin=get_origin(path),
git_commit_date=get_commit_date(path),
is_dirty=is_dirty(path)))
return None
cloudlog.exception("unable to get build metadata")
raise Exception("invalid build metadata")
if __name__ == "__main__":
@@ -122,12 +150,4 @@ if __name__ == "__main__":
params.put("TermsVersion", terms_version)
params.put("TrainingVersion", training_version)
print(f"Dirty: {is_dirty()}")
print(f"Version: {get_version()}")
print(f"Short version: {get_short_version()}")
print(f"Origin: {get_origin()}")
print(f"Normalized origin: {get_normalized_origin()}")
print(f"Branch: {get_branch()}")
print(f"Short branch: {get_short_branch()}")
print(f"Prebuilt: {is_prebuilt()}")
print(f"Commit date: {get_commit_date()}")
print(get_build_metadata())