Unverified Commit 6890cef0 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Use fixed encoding for our 'id' types (#100764)

Varint encoding works well for things like lengths/indicies which are
naturally 'zero biased' but many of our 'id' types are not like that

Switch them to use a fixed encoding

This includes: 

* `TaskId` mostly 5 bytes before, now 4 bytes always
* `CellId` always 5 bytes before, now 4 bytes always
* `RawVc` always 9 bytes before, 8 bytes now
* `FunctionId` mostly 3 bytes before, now 2 bytes
* `ValueTypeId` mostly 3 bytes before now 2 bytes
* `TraitTypeId` transformed to a `NonZeroU8` so it always takes the
fixed int encode path naturally

In principle it is possible that we might still benefit from varint
encoding if most of the uses of these ids were biased towards smaller
values, but at least in practice this isn't the case.

### Results

Measured on a large app comparing to canary

| | canary | this PR | change | per-round range |
|---|---|---|---|---|
| **Cache size after cold build** | 2.646 GB | 2.546 GB | **−3.8%** |
−3.8 to −3.7% |
| **Cache size after warm build** (compacted) | 2.628 GB | 2.527 GB |
**−3.8%** | −3.9 to −3.7% |
| **Warm build user CPU** (mostly decoding) | 43.8s | 42.2s | **−3.7%**
| −5.1 to −2.3% |
| Cold build user CPU | 482.6s | 483.0s | +0.1% | −1.4 to +1.0% |
| Wall time, cold / warm | 75.3s / 31.2s | 76.1s / 31.4s | within noise
| |
| Peak RSS, cold / warm | | | −0.2% / −0.8% | |
parent 778966e9
//! Fixed-width little-endian integer encoding, independent of the encoder's configured
//! [`IntEncoding`][bincode::config::IntEncoding]. This is useful when certain integers are
//! unlikely to benefit from varint encoding.
//!
//! The encoding here is consistent and uses LittleEndian encoding, ideally we would respect the
//! bincode settings but this isn't possible.
//!
//! [`TURBO_BINCODE_CONFIG`][crate::TURBO_BINCODE_CONFIG] uses varint encoding, which only saves
//! space for values below 2^16 (1 or 3 bytes) and costs an extra discriminant byte above that.
//! Identifiers such as task ids and packed cell ids are usually large, so they encode smaller (and
//! faster) at their natural width.
use bincode::{
de::{Decoder, read::Reader},
enc::{Encoder, write::Writer},
error::{DecodeError, EncodeError},
};
macro_rules! fixed_width {
($($ty:ty: $encode:ident, $decode:ident;)*) => {$(
#[doc = concat!("Encodes a `", stringify!($ty), "` as its little-endian bytes.")]
#[inline]
pub fn $encode<E: Encoder>(encoder: &mut E, value: $ty) -> Result<(), EncodeError> {
encoder.writer().write(&value.to_le_bytes())
}
#[doc = concat!("Decodes a `", stringify!($ty), "` written by [`", stringify!($encode), "`].")]
#[inline]
pub fn $decode<D: Decoder>(decoder: &mut D) -> Result<$ty, DecodeError> {
let mut bytes = [0u8; size_of::<$ty>()];
decoder.claim_bytes_read(bytes.len())?;
decoder.reader().read(&mut bytes)?;
Ok(<$ty>::from_le_bytes(bytes))
}
)*};
}
fixed_width! {
u8: encode_u8, decode_u8;
u16: encode_u16, decode_u16;
u32: encode_u32, decode_u32;
u64: encode_u64, decode_u64;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{TurboBincodeBuffer, new_turbo_bincode_decoder, new_turbo_bincode_encoder};
#[test]
fn roundtrip_at_natural_width() {
let mut buf = TurboBincodeBuffer::new();
let mut encoder = new_turbo_bincode_encoder(&mut buf);
encode_u8(&mut encoder, 3).unwrap();
encode_u16(&mut encoder, 7).unwrap();
encode_u32(&mut encoder, u32::MAX).unwrap();
encode_u64(&mut encoder, 1 << 40).unwrap();
assert_eq!(buf.len(), 1 + 2 + 4 + 8);
let mut decoder = new_turbo_bincode_decoder(&buf);
assert_eq!(decode_u8(&mut decoder).unwrap(), 3);
assert_eq!(decode_u16(&mut decoder).unwrap(), 7);
assert_eq!(decode_u32(&mut decoder).unwrap(), u32::MAX);
assert_eq!(decode_u64(&mut decoder).unwrap(), 1 << 40);
assert!(decode_u16(&mut decoder).is_err());
}
}
pub mod fixed_width;
mod len_prefixed; mod len_prefixed;
#[doc(hidden)] #[doc(hidden)]
pub mod macro_helpers; pub mod macro_helpers;
......
...@@ -9,10 +9,11 @@ use bincode::{ ...@@ -9,10 +9,11 @@ use bincode::{
Decode, Encode, Decode, Encode,
de::Decoder, de::Decoder,
enc::Encoder, enc::Encoder,
error::{DecodeError, EncodeError}, error::{DecodeError, EncodeError, IntegerType},
impl_borrow_decode, impl_borrow_decode,
}; };
use serde::{Deserialize, Serialize, de::Visitor}; use serde::{Deserialize, Serialize, de::Visitor};
use turbo_bincode::fixed_width;
use crate::{TaskPersistence, registry}; use crate::{TaskPersistence, registry};
...@@ -139,7 +140,7 @@ define_id!( ...@@ -139,7 +140,7 @@ define_id!(
TaskId: u32, TaskId: u32,
// Capped below `u32::MAX` so the id fits in 31 bits when packed into `RawVc`. // Capped below `u32::MAX` so the id fits in 31 bits when packed into `RawVc`.
max = TASK_ID_MAX, max = TASK_ID_MAX,
derive(Deserialize, Decode), derive(Deserialize),
serde(transparent), serde(transparent),
); );
define_id!( define_id!(
...@@ -148,7 +149,8 @@ define_id!( ...@@ -148,7 +149,8 @@ define_id!(
max = crate::CellId::MAX_VALUE_TYPE_ID, max = crate::CellId::MAX_VALUE_TYPE_ID,
); );
define_id!(FunctionId: u16); define_id!(FunctionId: u16);
define_id!(TraitTypeId: u16); // There are fewer than 100 traits, so a `u8` leaves plenty of headroom and encodes in 1 byte.
define_id!(TraitTypeId: u8);
define_id!( define_id!(
LocalTaskId: u32, LocalTaskId: u32,
derive(Debug, Serialize, Deserialize, Encode, Decode), derive(Debug, Serialize, Deserialize, Encode, Decode),
...@@ -163,8 +165,8 @@ define_id!( ...@@ -163,8 +165,8 @@ define_id!(
leak. This value may overflow and re-use old values.", leak. This value may overflow and re-use old values.",
); );
// Preserve transparent serde and the derived NonZero<u32> bincode format, but reject references // Preserve transparent serde, but reject references to transient tasks. RawVc packs TaskId into its
// to transient tasks. RawVc packs TaskId into its own integer and checks that path separately. // own integer and checks that path separately.
impl Serialize for TaskId { impl Serialize for TaskId {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
debug_assert!( debug_assert!(
...@@ -175,13 +177,25 @@ impl Serialize for TaskId { ...@@ -175,13 +177,25 @@ impl Serialize for TaskId {
} }
} }
// Task ids are allocated sequentially and quickly exceed 2^16, where a varint takes 5 bytes. Encode
// them at their natural width instead.
impl Encode for TaskId { impl Encode for TaskId {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> { fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
debug_assert!(!self.is_transient(), "transient TaskId must not be encoded"); debug_assert!(!self.is_transient(), "transient TaskId must not be encoded");
self.id.encode(encoder) fixed_width::encode_u32(encoder, self.id.get())
} }
} }
impl<Context> Decode<Context> for TaskId {
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
Self::new(fixed_width::decode_u32(decoder)?).ok_or(DecodeError::NonZeroTypeIsZero {
non_zero_type: IntegerType::U32,
})
}
}
impl_borrow_decode!(TaskId);
impl Debug for TaskId { impl Debug for TaskId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TaskId").field("id", &self.id).finish() f.debug_struct("TaskId").field("id", &self.id).finish()
...@@ -212,13 +226,20 @@ impl TaskId { ...@@ -212,13 +226,20 @@ impl TaskId {
} }
macro_rules! make_registered_serializable { macro_rules! make_registered_serializable {
($ty:ty, $primitive:ty, $get_object:path, $validate_type_id:path $(,)?) => { (
$ty:ty,
$primitive:ty,
$get_object:path,
$validate_type_id:path,
$encode_primitive:path,
$decode_primitive:path $(,)?
) => {
impl Serialize for $ty { impl Serialize for $ty {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where where
S: serde::Serializer, S: serde::Serializer,
{ {
serializer.serialize_u16(self.id.into()) serializer.serialize_u16(self.id.get().into())
} }
} }
...@@ -239,7 +260,7 @@ macro_rules! make_registered_serializable { ...@@ -239,7 +260,7 @@ macro_rules! make_registered_serializable {
where where
E: serde::de::Error, E: serde::de::Error,
{ {
match Self::Value::new(v) { match <$primitive>::try_from(v).ok().and_then(Self::Value::new) {
Some(value) => { Some(value) => {
if let Some(error) = $validate_type_id(value) { if let Some(error) = $validate_type_id(value) {
Err(E::custom(error)) Err(E::custom(error))
...@@ -247,7 +268,10 @@ macro_rules! make_registered_serializable { ...@@ -247,7 +268,10 @@ macro_rules! make_registered_serializable {
Ok(value) Ok(value)
} }
} }
None => Err(E::unknown_variant(&format!("{v}"), &["a non zero u16"])), None => Err(E::unknown_variant(
&format!("{v}"),
&[concat!("a non zero ", stringify!($primitive))],
)),
} }
} }
} }
...@@ -267,15 +291,14 @@ macro_rules! make_registered_serializable { ...@@ -267,15 +291,14 @@ macro_rules! make_registered_serializable {
impl Encode for $ty { impl Encode for $ty {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> { fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
<NonZero<$primitive> as Encode>::encode(&self.id, encoder) $encode_primitive(encoder, self.id.get())
} }
} }
impl<Context> Decode<Context> for $ty { impl<Context> Decode<Context> for $ty {
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> { fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
let value = Self { let value = Self::new($decode_primitive(decoder)?)
id: NonZero::<$primitive>::decode(decoder)?, .ok_or(DecodeError::Other(concat!("zero ", stringify!($ty))))?;
};
if let Some(error) = $validate_type_id(value) { if let Some(error) = $validate_type_id(value) {
Err(DecodeError::OtherString(error.to_string())) Err(DecodeError::OtherString(error.to_string()))
} else { } else {
...@@ -288,21 +311,30 @@ macro_rules! make_registered_serializable { ...@@ -288,21 +311,30 @@ macro_rules! make_registered_serializable {
}; };
} }
// Registry ids are assigned in sorted name order, so their size is unrelated to how often they are
// used. There are well over 250 value types and functions, where a varint grows to 3 bytes, so
// encode them at their natural width.
make_registered_serializable!( make_registered_serializable!(
ValueTypeId, ValueTypeId,
u16, u16,
registry::get_value_type, registry::get_value_type,
registry::validate_value_type_id, registry::validate_value_type_id,
fixed_width::encode_u16,
fixed_width::decode_u16,
); );
make_registered_serializable!( make_registered_serializable!(
TraitTypeId, TraitTypeId,
u16, u8,
registry::get_trait, registry::get_trait,
registry::validate_trait_type_id, registry::validate_trait_type_id,
fixed_width::encode_u8,
fixed_width::decode_u8,
); );
make_registered_serializable!( make_registered_serializable!(
FunctionId, FunctionId,
u16, u16,
registry::get_native_function, registry::get_native_function,
registry::validate_function_id, registry::validate_function_id,
fixed_width::encode_u16,
fixed_width::decode_u16,
); );
...@@ -115,7 +115,7 @@ pub trait RegistryDef<T: 'static> { ...@@ -115,7 +115,7 @@ pub trait RegistryDef<T: 'static> {
/// while maintaining their specific requirements. /// while maintaining their specific requirements.
trait Registerable: 'static + Eq + std::hash::Hash { trait Registerable: 'static + Eq + std::hash::Hash {
/// The ID type used for this registry item /// The ID type used for this registry item
type Id: Copy + From<NonZeroU16> + std::ops::Deref<Target = u16> + std::fmt::Display; type Id: Copy + std::ops::Deref<Target: Copy + Into<usize>> + std::fmt::Display;
const TYPE_NAME: &'static str; const TYPE_NAME: &'static str;
/// The largest id that may be assigned to this registry item. /// The largest id that may be assigned to this registry item.
...@@ -123,6 +123,13 @@ trait Registerable: 'static + Eq + std::hash::Hash { ...@@ -123,6 +123,13 @@ trait Registerable: 'static + Eq + std::hash::Hash {
/// Get the global registry type used for sorting and uniqueness validation /// Get the global registry type used for sorting and uniqueness validation
fn ty(&self) -> &RegistryType; fn ty(&self) -> &RegistryType;
/// Converts an id assigned by [`init_registry`] into [`Self::Id`].
///
/// # Safety
///
/// `id` must be at most [`Self::MAX_ID`].
unsafe fn id_from_raw(id: NonZeroU16) -> Self::Id;
} }
impl Registerable for NativeFunction { impl Registerable for NativeFunction {
...@@ -132,6 +139,10 @@ impl Registerable for NativeFunction { ...@@ -132,6 +139,10 @@ impl Registerable for NativeFunction {
fn ty(&self) -> &RegistryType { fn ty(&self) -> &RegistryType {
&self.ty &self.ty
} }
unsafe fn id_from_raw(id: NonZeroU16) -> FunctionId {
FunctionId::from(id)
}
} }
impl Registerable for ValueType { impl Registerable for ValueType {
...@@ -141,14 +152,24 @@ impl Registerable for ValueType { ...@@ -141,14 +152,24 @@ impl Registerable for ValueType {
fn ty(&self) -> &RegistryType { fn ty(&self) -> &RegistryType {
&self.ty &self.ty
} }
unsafe fn id_from_raw(id: NonZeroU16) -> ValueTypeId {
ValueTypeId::from(id)
}
} }
impl Registerable for TraitType { impl Registerable for TraitType {
type Id = TraitTypeId; type Id = TraitTypeId;
const TYPE_NAME: &'static str = "Trait"; const TYPE_NAME: &'static str = "Trait";
const MAX_ID: u16 = TraitTypeId::MAX.to_primitive() as u16;
fn ty(&self) -> &RegistryType { fn ty(&self) -> &RegistryType {
&self.ty &self.ty
} }
unsafe fn id_from_raw(id: NonZeroU16) -> TraitTypeId {
// SAFETY: the caller guarantees `id <= MAX_ID`, which is `u8::MAX`.
unsafe { TraitTypeId::new_unchecked(id.get() as u8) }
}
} }
/// Assign IDs to items and call post_init. Shared logic for all registry types. /// Assign IDs to items and call post_init. Shared logic for all registry types.
...@@ -185,7 +206,7 @@ fn init_registry<T: Registerable>(mut items: Vec<&'static T>) -> Box<[&'static T ...@@ -185,7 +206,7 @@ fn init_registry<T: Registerable>(mut items: Vec<&'static T>) -> Box<[&'static T
/// Get an item by its ID from a registry slice /// Get an item by its ID from a registry slice
#[inline] #[inline]
fn get_item<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, id: T::Id) -> &'static T { fn get_item<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, id: T::Id) -> &'static T {
registry[*id as usize - 1] registry[(*id).into() - 1]
} }
/// Read an item's assigned id directly, without touching `LazyLock`. /// Read an item's assigned id directly, without touching `LazyLock`.
...@@ -205,7 +226,8 @@ unsafe fn get_id_unchecked<T: Registerable>(item: &'static T) -> T::Id { ...@@ -205,7 +226,8 @@ unsafe fn get_id_unchecked<T: Registerable>(item: &'static T) -> T::Id {
item = item.ty().global_name item = item.ty().global_name
); );
}; };
T::Id::from(id) // SAFETY: `init_registry` only writes ids up to `T::MAX_ID`.
unsafe { T::id_from_raw(id) }
} }
/// Get the ID for a registered item. Forces registry init if needed, which /// Get the ID for a registered item. Forces registry init if needed, which
...@@ -223,7 +245,7 @@ fn validate_id<T: Registerable>( ...@@ -223,7 +245,7 @@ fn validate_id<T: Registerable>(
id: T::Id, id: T::Id,
) -> Option<Error> { ) -> Option<Error> {
let len = registry.len(); let len = registry.len();
if *id as usize <= len { if (*id).into() <= len {
None None
} else { } else {
Some(anyhow::anyhow!( Some(anyhow::anyhow!(
......
...@@ -342,7 +342,8 @@ impl TraitMethod { ...@@ -342,7 +342,8 @@ impl TraitMethod {
// SAFETY: Written during single-threaded Lazy init. Lazy provides acquire barrier. // SAFETY: Written during single-threaded Lazy init. Lazy provides acquire barrier.
let raw = unsafe { std::ptr::read(self.trait_type.ty.id.get()) }; let raw = unsafe { std::ptr::read(self.trait_type.ty.id.get()) };
debug_assert!(raw != 0, "TraitMethod::trait_type_id not initialized"); debug_assert!(raw != 0, "TraitMethod::trait_type_id not initialized");
unsafe { TraitTypeId::new_unchecked(raw) } // The registry caps trait ids at `TraitTypeId::MAX`, so this cannot truncate.
unsafe { TraitTypeId::new_unchecked(raw as u8) }
} }
pub(crate) fn resolve_span(&self, priority: TaskPriority) -> Span { pub(crate) fn resolve_span(&self, priority: TaskPriority) -> Span {
......
...@@ -8,8 +8,15 @@ use std::{ ...@@ -8,8 +8,15 @@ use std::{
}; };
use anyhow::Result; use anyhow::Result;
use bincode::{Decode, Encode, enc::Encoder, error::EncodeError}; use bincode::{
Decode, Encode,
de::Decoder,
enc::Encoder,
error::{DecodeError, EncodeError, IntegerType},
impl_borrow_decode,
};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_bincode::fixed_width;
use crate::{ use crate::{
ReadCellOptions, ReadConsistency, ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId, ReadCellOptions, ReadConsistency, ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId,
...@@ -36,9 +43,32 @@ use crate::{ ...@@ -36,9 +43,32 @@ use crate::{
/// bits 21..=0 (22 bits): cell index /// bits 21..=0 (22 bits): cell index
/// ``` /// ```
/// Because the `ValueTypeId` is always `>= 1`, the type is trivially non-zero /// Because the `ValueTypeId` is always `>= 1`, the type is trivially non-zero
#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode)] #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct CellId(NonZeroU32); pub struct CellId(NonZeroU32);
// The `ValueTypeId` occupies the high bits, so the packed word is always `>= 2^22` and a varint
// would always take 5 bytes.
impl Encode for CellId {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
fixed_width::encode_u32(encoder, self.raw())
}
}
impl<Context> Decode<Context> for CellId {
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
let raw = fixed_width::decode_u32(decoder)?;
if raw >> CELL_INDEX_BITS == 0 {
return Err(DecodeError::NonZeroTypeIsZero {
non_zero_type: IntegerType::U32,
});
}
// SAFETY: the type id bits are non-zero, which is all `from_raw` requires.
Ok(unsafe { Self::from_raw(raw) })
}
}
impl_borrow_decode!(CellId);
/// Number of low bits used for the cell index. /// Number of low bits used for the cell index.
const CELL_INDEX_BITS: u32 = 22; const CELL_INDEX_BITS: u32 = 22;
/// Mask selecting the cell index bits. /// Mask selecting the cell index bits.
...@@ -143,11 +173,11 @@ impl Display for CellId { ...@@ -143,11 +173,11 @@ impl Display for CellId {
/// ``` /// ```
/// [`Vc`]: crate::Vc /// [`Vc`]: crate::Vc
/// [monomorphization]: https://doc.rust-lang.org/book/ch10-01-syntax.html#performance-of-code-using-generics /// [monomorphization]: https://doc.rust-lang.org/book/ch10-01-syntax.html#performance-of-code-using-generics
#[derive(Clone, Copy, PartialEq, Eq, Hash, Deserialize, Decode)] #[derive(Clone, Copy, PartialEq, Eq, Hash, Deserialize)]
pub struct RawVc(NonZeroU64); pub struct RawVc(NonZeroU64);
// RawVc serializes its packed integer directly, so TaskId's guards cannot check embedded task // RawVc serializes its packed integer directly, so TaskId's guards cannot check embedded task
// ids. Keep the original serde newtype shape and bincode format while rejecting transient values. // ids. Keep the original serde newtype shape while rejecting transient values.
impl Serialize for RawVc { impl Serialize for RawVc {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
debug_assert!( debug_assert!(
...@@ -158,13 +188,27 @@ impl Serialize for RawVc { ...@@ -158,13 +188,27 @@ impl Serialize for RawVc {
} }
} }
// Encoded `RawVc`s are almost always `TaskCell`s, whose high bits hold a `CellId`, so a varint
// would take 9 bytes. `OperationVc` (always `TaskOutput`) stores a bare `TaskId`, not a `RawVc`.
impl Encode for RawVc { impl Encode for RawVc {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> { fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
debug_assert!(!self.is_transient(), "transient RawVc must not be encoded"); debug_assert!(!self.is_transient(), "transient RawVc must not be encoded");
self.0.encode(encoder) fixed_width::encode_u64(encoder, self.bits())
} }
} }
impl<Context> Decode<Context> for RawVc {
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
NonZeroU64::new(fixed_width::decode_u64(decoder)?)
.map(RawVc)
.ok_or(DecodeError::NonZeroTypeIsZero {
non_zero_type: IntegerType::U64,
})
}
}
impl_borrow_decode!(RawVc);
/// The unpacked form of [`RawVc`], produced by [`RawVc::unpack`]. /// The unpacked form of [`RawVc`], produced by [`RawVc::unpack`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum RawVcUnpacked { pub enum RawVcUnpacked {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment