Created
November 17, 2022 02:22
-
-
Save sschr15/6015391bfe088ae2ee9e23b87a220a86 to your computer and use it in GitHub Desktop.
A Python implementation of the data structures used for @skyrising's mc-versions project
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import annotations | |
from dataclasses import dataclass, field | |
from datetime import datetime | |
from enum import Enum, auto | |
from typing import Optional, Union | |
import requests | |
class VersionType(Enum): | |
release = auto() | |
snapshot = auto() | |
old_beta = auto() | |
old_alpha = auto() | |
alpha_server = auto() | |
classic_server = auto() | |
pending = auto() | |
@classmethod | |
def from_string(cls, string: str) -> VersionType: | |
return { | |
"release": cls.release, | |
"snapshot": cls.snapshot, | |
"old_beta": cls.old_beta, | |
"old_alpha": cls.old_alpha, | |
"alpha_server": cls.alpha_server, | |
"classic_server": cls.classic_server, | |
"pending": cls.pending, | |
}[string] | |
class WorldFormat(Enum): | |
classic = auto() | |
indev = auto() | |
alpha = auto() | |
mcregion = auto() | |
anvil = auto() | |
@classmethod | |
def from_string(cls, string: str) -> WorldFormat: | |
return { | |
"classic": cls.classic, | |
"indev": cls.indev, | |
"alpha": cls.alpha, | |
"region": cls.mcregion, | |
"anvil": cls.anvil, | |
}[string] | |
@dataclass | |
class Download: | |
sha1: str | |
size: int | |
url: str | |
@classmethod | |
def from_dict(cls, data: dict) -> Download: | |
return cls(data["sha1"], data["size"], data["url"]) | |
@dataclass | |
class ExtraDataManifest: | |
asset_hash: Optional[str] | |
asset_index: Optional[str] | |
downloads_hash: Optional[str] | |
downloads_id: int | |
hash: str | |
last_modified: Optional[datetime] | |
time: Optional[datetime] | |
type: VersionType | |
url: str | |
@classmethod | |
def from_dict(cls, data: dict) -> ExtraDataManifest: | |
return cls( | |
data["assetHash"] if "assetHash" in data else None, | |
data["assetIndex"] if "assetIndex" in data else None, | |
data["downloadsHash"] if "downloadsHash" in data else None, | |
data["downloadsId"], | |
data["hash"], | |
datetime.fromisoformat(data["lastModified"]) if "lastModified" in data else None, | |
datetime.fromisoformat(data["time"]) if "time" in data else None, | |
VersionType.from_string(data["type"]), | |
data["url"], | |
) | |
@dataclass | |
class Protocol: | |
type: str | |
version: int | |
@classmethod | |
def from_dict(cls, data: dict) -> Protocol: | |
return cls(data["type"], data["version"]) | |
@dataclass | |
class ExtraVersionInfo: | |
is_client: bool | |
downloads: dict[str, Download] | |
id: str | |
manifests: list[ExtraDataManifest] | |
next_versions: list[str] | |
normalized_version: str | |
previous_versions: list[str] | |
protocol: Optional[Protocol] | |
release_target: Optional[str] | |
release_time: datetime | |
is_server: bool | |
shared_mappings: bool | |
world_format: Optional[WorldFormat] | |
@classmethod | |
def from_dict(cls, data: dict) -> ExtraVersionInfo: | |
return cls( | |
data["client"], | |
{key: Download.from_dict(value) for key, value in data["downloads"].items()}, | |
data["id"], | |
[ExtraDataManifest.from_dict(value) for value in data["manifests"]], | |
data["next"], | |
data["normalizedVersion"], | |
data["previous"], | |
Protocol.from_dict(data["protocol"]) if data.get("protocol") else None, # the first instance of a null value in the json! | |
data["releaseTarget"] if "releaseTarget" in data else None, | |
datetime.fromisoformat(data["releaseTime"]), | |
data["server"], | |
data["sharedMappings"], | |
WorldFormat.from_string(data["world"]["format"]) if "world" in data else None, | |
) | |
@dataclass | |
class OSRule: | |
name: Optional[str] | |
version: Optional[str] | |
arch: Optional[str] | |
@classmethod | |
def from_dict(cls, data: dict) -> OSRule: | |
return cls(data.get("name"), data.get("version"), data.get("arch")) | |
@dataclass | |
class Rule: | |
allow: bool | |
features: Optional[dict[str, bool]] | |
os: Optional[OSRule] | |
@classmethod | |
def from_dict(cls, data: dict) -> Rule: | |
return cls( | |
data["action"] == "allow", | |
data.get("features"), | |
OSRule.from_dict(data["os"]) if "os" in data else None, | |
) | |
@dataclass | |
class JvmArgument: | |
value: Union[str, list[str]] | |
rules: list[Rule] | |
@classmethod | |
def from_dict(cls, data: dict) -> JvmArgument: | |
return cls(data["value"], [Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else []) | |
@dataclass | |
class JvmArguments: | |
game: list[Union[str, JvmArgument]] | |
jvm: list[Union[str, JvmArgument]] | |
@classmethod | |
def from_dict(cls, data: dict) -> JvmArguments: | |
game_data = data["game"] | |
jvm_data = data["jvm"] | |
for i, arg in enumerate(game_data): | |
if isinstance(arg, dict): | |
game_data[i] = JvmArgument.from_dict(arg) | |
for i, arg in enumerate(jvm_data): | |
if isinstance(arg, dict): | |
jvm_data[i] = JvmArgument.from_dict(arg) | |
return cls(game_data, jvm_data) | |
@dataclass | |
class AssetIndex: | |
id: str | |
sha1: str | |
size: int | |
total_size: int | |
url: str | |
@classmethod | |
def from_dict(cls, data: dict) -> AssetIndex: | |
return cls(data["id"], data["sha1"], data["size"], data["totalSize"], data["url"]) | |
@dataclass | |
class JavaVersion: | |
component: str | |
major_version: int | |
@classmethod | |
def from_dict(cls, data: dict) -> JavaVersion: | |
return cls(data["component"], data["majorVersion"]) | |
# Stub to use in the Library class then actually define later | |
def native_lib(data: dict) -> Library: ... # type: ignore | |
def maven_lib(data: dict) -> Library: ... # type: ignore | |
@dataclass | |
class Library: | |
path: str | |
sha1: str | |
size: int | |
url: str | |
name: str | |
rules: list[Rule] | |
@classmethod | |
def from_dict(cls, data: dict) -> Library: | |
if "natives" in data: | |
return native_lib(data) | |
elif "downloads" not in data: | |
return maven_lib(data) | |
return cls( | |
data["downloads"]["artifact"]["path"], | |
data["downloads"]["artifact"]["sha1"], | |
data["downloads"]["artifact"]["size"], | |
data["downloads"]["artifact"]["url"], | |
data["name"], | |
[Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else [], | |
) | |
@dataclass | |
class NativeLibrary(Library): | |
natives: dict[str, str] | |
classifiers: Optional[dict[str, Download]] | |
extract: Optional[dict[str, str]] | |
@classmethod | |
def from_dict(cls, data: dict) -> NativeLibrary: | |
natives = data["natives"] | |
classifiers = {k: Download.from_dict(v) for k, v in data["downloads"]["classifiers"].items()} if "downloads" in data and "classifiers" in data["downloads"] else None | |
classifier = next(iter(classifiers.values())) if classifiers else None | |
return cls( | |
"", | |
classifier.sha1 if classifier else "", | |
classifier.size if classifier else 0, | |
classifier.url if classifier else "", | |
data["name"], | |
[Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else [], | |
natives, | |
classifiers, | |
data["extract"] if "extract" in data else None, | |
) | |
@dataclass | |
class MavenLibrary(Library): | |
@classmethod | |
def from_dict(cls, data: dict) -> MavenLibrary: | |
return cls( | |
"", | |
"", | |
0, | |
"", | |
data["name"], | |
[Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else [], | |
) | |
def native_lib(data: dict) -> NativeLibrary: | |
return NativeLibrary.from_dict(data) | |
def maven_lib(data: dict) -> MavenLibrary: | |
return MavenLibrary.from_dict(data) | |
class LoggingSide(Enum): | |
client = auto() | |
server = auto() | |
@classmethod | |
def from_string(cls, string: str) -> LoggingSide: | |
return { | |
"client": cls.client, | |
"server": cls.server, | |
}[string] | |
@dataclass | |
class Logging: | |
argument: str | |
file_id: str | |
file_sha1: str | |
file_size: int | |
file_url: str | |
type: str | |
@classmethod | |
def from_dict(cls, data: dict) -> Logging: | |
return cls( | |
data["argument"], | |
data["file"]["id"], | |
data["file"]["sha1"], | |
data["file"]["size"], | |
data["file"]["url"], | |
data["type"], | |
) | |
@dataclass | |
class Manifest: | |
arguments: Optional[JvmArguments] | |
asset_index: Optional[AssetIndex] | |
assets: Optional[str] | |
compliance_level: Optional[int] | |
downloads: dict[str, Download] | |
id: str | |
java_version: Optional[JavaVersion] | |
libraries: list[Library] | |
logging: Optional[dict[LoggingSide, Logging]] | |
main_class: Optional[str] | |
minimum_launcher_version: Optional[int] | |
release_time: datetime | |
time: Optional[datetime] | |
type: VersionType | |
@classmethod | |
def from_dict(cls, data: dict) -> Manifest: | |
return cls( | |
JvmArguments.from_dict(data["arguments"]) if "arguments" in data else None, | |
AssetIndex.from_dict(data["assetIndex"]) if "assetIndex" in data else None, | |
data["assets"] if "assets" in data else None, | |
data["complianceLevel"] if "complianceLevel" in data else None, | |
{key: Download.from_dict(value) for key, value in data["downloads"].items()}, | |
data["id"], | |
JavaVersion.from_dict(data["javaVersion"]) if "javaVersion" in data else None, | |
[Library.from_dict(value) for value in data["libraries"]], | |
{LoggingSide.from_string(key): Logging.from_dict(value) for key, value in data["logging"].items()} if "logging" in data else None, | |
data["mainClass"] if "mainClass" in data else None, | |
data["minimumLauncherVersion"] if "minimumLauncherVersion" in data else None, | |
datetime.fromisoformat(data["releaseTime"].replace("T24:", "T00:")), # Fix for Classic c0.27_st | |
datetime.fromisoformat(data["time"]) if "time" in data else None, | |
VersionType.from_string(data["type"]), | |
) | |
@dataclass | |
class Version: | |
id: str | |
omni_id: str | |
type: VersionType | |
url: str | |
time: Optional[datetime] | |
release_time: datetime | |
details_url: str | |
_manifest: Optional[Manifest] = field(init=False, repr=False, default=None) | |
_details: Optional[ExtraVersionInfo] = field(init=False, repr=False, default=None) | |
@property | |
def manifest(self) -> Manifest: | |
"""Lazy load the manifest.""" | |
if self._manifest is None: | |
if "%" in self.url: | |
url = self.url.replace("%", "%25") # funky | |
else: | |
url = self.url | |
with requests.get(url) as response: | |
self._manifest = Manifest.from_dict(response.json()) | |
return self._manifest | |
@property | |
def details(self) -> ExtraVersionInfo: | |
"""Lazy load extra version info.""" | |
if self._details is None: | |
with requests.get(self.details_url) as response: | |
self._details = ExtraVersionInfo.from_dict(response.json()) | |
return self._details | |
@classmethod | |
def from_dict(cls, data: dict) -> Version: | |
return cls( | |
data["id"], | |
data["omniId"], | |
VersionType.from_string(data["type"]), | |
data["url"], | |
datetime.fromisoformat(data["time"]) if "time" in data else None, | |
datetime.fromisoformat(data["releaseTime"]), | |
data["details"], | |
) | |
@dataclass | |
class VersionManifest: | |
latest: dict[str, str] | |
versions: list[Version] | |
_versions_by_id: dict[str, Version] = field(init=False, default_factory=dict) | |
def __getitem__(self, name: str) -> Optional[Version]: | |
if name in self._versions_by_id: | |
return self._versions_by_id[name] | |
for version in self.versions: | |
if version.omni_id == name: | |
self._versions_by_id[name] = version | |
return version | |
return None | |
@classmethod | |
def from_dict(cls, data: dict) -> VersionManifest: | |
instance = cls( | |
data["latest"], | |
[Version.from_dict(version) for version in data["versions"]], | |
) | |
for version in instance.latest.values(): | |
instance[version] # Add all the latest versions to the cache | |
return instance | |
def get_version_manifest(root_url: str) -> VersionManifest: | |
"""Get the version manifest.""" | |
with requests.get(root_url + "version_manifest.json") as response: | |
return VersionManifest.from_dict(response.json()) | |
def test(): | |
from os import path | |
manifest = get_version_manifest("https://skyrising.github.io/mc-versions/") | |
if path.exists("last"): | |
with open("last") as file: | |
last_version = file.read() | |
skip = True | |
else: | |
last_version = None | |
skip = False | |
try: | |
for version in manifest.versions: | |
if version.id == last_version: | |
skip = False | |
print(f'(Skipping {version.id} because it was the most recent version last time)') | |
if skip: | |
print(f'(Skipping {version.id} because it was already processed)') | |
continue | |
# Double-check that both the manifest and details can be loaded | |
version.manifest | |
version.details | |
print(f'Checked {version.id} ({version.type})') | |
last_version = version.id | |
except Exception as e: | |
with open("last", "w") as f: | |
f.write(str(last_version)) | |
print(f'Picking up at {last_version} next time') | |
raise e | |
if __name__ == "__main__": | |
test() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment