| Alphare |
| hg-reviewers |
This can be later extended to access metadata such as uncompressed_len without
necessarily resolving deltas.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | rust/hg-core/src/revlog/filelog.rs (32 lines) | |||
| M | rust/hg-core/src/revlog/revlog.rs (49 lines) | |||
| M | rust/rhg/src/commands/status.rs (11 lines) |
| Status | Author | Revision | |
|---|---|---|---|
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin |
| use crate::errors::HgError; | use crate::errors::HgError; | ||||
| use crate::repo::Repo; | use crate::repo::Repo; | ||||
| use crate::revlog::path_encode::path_encode; | use crate::revlog::path_encode::path_encode; | ||||
| use crate::revlog::revlog::RevlogEntry; | |||||
| use crate::revlog::revlog::{Revlog, RevlogError}; | use crate::revlog::revlog::{Revlog, RevlogError}; | ||||
| use crate::revlog::NodePrefix; | use crate::revlog::NodePrefix; | ||||
| use crate::revlog::Revision; | use crate::revlog::Revision; | ||||
| use crate::utils::files::get_path_from_bytes; | use crate::utils::files::get_path_from_bytes; | ||||
| use crate::utils::hg_path::HgPath; | use crate::utils::hg_path::HgPath; | ||||
| use crate::utils::SliceExt; | use crate::utils::SliceExt; | ||||
| use std::path::PathBuf; | use std::path::PathBuf; | ||||
| /// A specialized `Revlog` to work with file data logs. | /// A specialized `Revlog` to work with file data logs. | ||||
| pub struct Filelog { | pub struct Filelog { | ||||
| /// The generic `revlog` format. | /// The generic `revlog` format. | ||||
| revlog: Revlog, | revlog: Revlog, | ||||
| } | } | ||||
| impl Filelog { | impl Filelog { | ||||
| pub fn open(repo: &Repo, file_path: &HgPath) -> Result<Self, HgError> { | pub fn open(repo: &Repo, file_path: &HgPath) -> Result<Self, HgError> { | ||||
| let index_path = store_path(file_path, b".i"); | let index_path = store_path(file_path, b".i"); | ||||
| let data_path = store_path(file_path, b".d"); | let data_path = store_path(file_path, b".d"); | ||||
| let revlog = Revlog::open(repo, index_path, Some(&data_path))?; | let revlog = Revlog::open(repo, index_path, Some(&data_path))?; | ||||
| Ok(Self { revlog }) | Ok(Self { revlog }) | ||||
| } | } | ||||
| /// The given node ID is that of the file as found in a manifest, not of a | /// The given node ID is that of the file as found in a filelog, not of a | ||||
| /// changeset. | /// changeset. | ||||
| pub fn data_for_node( | pub fn data_for_node( | ||||
| &self, | &self, | ||||
| file_node: impl Into<NodePrefix>, | file_node: impl Into<NodePrefix>, | ||||
| ) -> Result<FilelogRevisionData, RevlogError> { | ) -> Result<FilelogRevisionData, RevlogError> { | ||||
| let file_rev = self.revlog.rev_from_node(file_node.into())?; | let file_rev = self.revlog.rev_from_node(file_node.into())?; | ||||
| self.data_for_rev(file_rev) | self.data_for_rev(file_rev) | ||||
| } | } | ||||
| /// The given revision is that of the file as found in a manifest, not of a | /// The given revision is that of the file as found in a filelog, not of a | ||||
| /// changeset. | /// changeset. | ||||
| pub fn data_for_rev( | pub fn data_for_rev( | ||||
| &self, | &self, | ||||
| file_rev: Revision, | file_rev: Revision, | ||||
| ) -> Result<FilelogRevisionData, RevlogError> { | ) -> Result<FilelogRevisionData, RevlogError> { | ||||
| let data: Vec<u8> = self.revlog.get_rev_data(file_rev)?.into_owned(); | let data: Vec<u8> = self.revlog.get_rev_data(file_rev)?.into_owned(); | ||||
| Ok(FilelogRevisionData(data.into())) | Ok(FilelogRevisionData(data.into())) | ||||
| } | } | ||||
| /// The given node ID is that of the file as found in a filelog, not of a | |||||
| /// changeset. | |||||
| pub fn entry_for_node( | |||||
| &self, | |||||
| file_node: impl Into<NodePrefix>, | |||||
| ) -> Result<FilelogEntry, RevlogError> { | |||||
| let file_rev = self.revlog.rev_from_node(file_node.into())?; | |||||
| self.entry_for_rev(file_rev) | |||||
| } | |||||
| /// The given revision is that of the file as found in a filelog, not of a | |||||
| /// changeset. | |||||
| pub fn entry_for_rev( | |||||
| &self, | |||||
| file_rev: Revision, | |||||
| ) -> Result<FilelogEntry, RevlogError> { | |||||
| Ok(FilelogEntry(self.revlog.get_entry(file_rev)?)) | |||||
| } | |||||
| } | } | ||||
| fn store_path(hg_path: &HgPath, suffix: &[u8]) -> PathBuf { | fn store_path(hg_path: &HgPath, suffix: &[u8]) -> PathBuf { | ||||
| let encoded_bytes = | let encoded_bytes = | ||||
| path_encode(&[b"data/", hg_path.as_bytes(), suffix].concat()); | path_encode(&[b"data/", hg_path.as_bytes(), suffix].concat()); | ||||
| get_path_from_bytes(&encoded_bytes).into() | get_path_from_bytes(&encoded_bytes).into() | ||||
| } | } | ||||
| pub struct FilelogEntry<'a>(RevlogEntry<'a>); | |||||
| impl FilelogEntry<'_> { | |||||
| pub fn data(&self) -> Result<FilelogRevisionData, HgError> { | |||||
| Ok(FilelogRevisionData(self.0.data()?.into_owned())) | |||||
| } | |||||
| } | |||||
| /// The data for one revision in a filelog, uncompressed and delta-resolved. | /// The data for one revision in a filelog, uncompressed and delta-resolved. | ||||
| pub struct FilelogRevisionData(Vec<u8>); | pub struct FilelogRevisionData(Vec<u8>); | ||||
| impl FilelogRevisionData { | impl FilelogRevisionData { | ||||
| /// Split into metadata and data | /// Split into metadata and data | ||||
| pub fn split(&self) -> Result<(Option<&[u8]>, &[u8]), HgError> { | pub fn split(&self) -> Result<(Option<&[u8]>, &[u8]), HgError> { | ||||
| const DELIMITER: &[u8; 2] = &[b'\x01', b'\n']; | const DELIMITER: &[u8; 2] = &[b'\x01', b'\n']; | ||||
| fn from(error: NodeMapError) -> Self { | fn from(error: NodeMapError) -> Self { | ||||
| match error { | match error { | ||||
| NodeMapError::MultipleResults => RevlogError::AmbiguousPrefix, | NodeMapError::MultipleResults => RevlogError::AmbiguousPrefix, | ||||
| NodeMapError::RevisionNotInIndex(_) => RevlogError::corrupted(), | NodeMapError::RevisionNotInIndex(_) => RevlogError::corrupted(), | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| fn corrupted() -> HgError { | |||||
| HgError::corrupted("corrupted revlog") | |||||
| } | |||||
| impl RevlogError { | impl RevlogError { | ||||
| fn corrupted() -> Self { | fn corrupted() -> Self { | ||||
| RevlogError::Other(HgError::corrupted("corrupted revlog")) | RevlogError::Other(corrupted()) | ||||
| } | } | ||||
| } | } | ||||
| /// Read only implementation of revlog. | /// Read only implementation of revlog. | ||||
| pub struct Revlog { | pub struct Revlog { | ||||
| /// When index and data are not interleaved: bytes of the revlog index. | /// When index and data are not interleaved: bytes of the revlog index. | ||||
| /// When index and data are interleaved: bytes of the revlog index and | /// When index and data are interleaved: bytes of the revlog index and | ||||
| /// data. | /// data. | ||||
| #[timed] | #[timed] | ||||
| pub fn get_rev_data( | pub fn get_rev_data( | ||||
| &self, | &self, | ||||
| rev: Revision, | rev: Revision, | ||||
| ) -> Result<Cow<[u8]>, RevlogError> { | ) -> Result<Cow<[u8]>, RevlogError> { | ||||
| if rev == NULL_REVISION { | if rev == NULL_REVISION { | ||||
| return Ok(Cow::Borrowed(&[])); | return Ok(Cow::Borrowed(&[])); | ||||
| }; | }; | ||||
| self.get_entry(rev)?.data() | Ok(self.get_entry(rev)?.data()?) | ||||
| } | } | ||||
| /// Check the hash of some given data against the recorded hash. | /// Check the hash of some given data against the recorded hash. | ||||
| pub fn check_hash( | pub fn check_hash( | ||||
| &self, | &self, | ||||
| p1: Revision, | p1: Revision, | ||||
| p2: Revision, | p2: Revision, | ||||
| expected: &[u8], | expected: &[u8], | ||||
| } | } | ||||
| /// Build the full data of a revision out its snapshot | /// Build the full data of a revision out its snapshot | ||||
| /// and its deltas. | /// and its deltas. | ||||
| #[timed] | #[timed] | ||||
| fn build_data_from_deltas( | fn build_data_from_deltas( | ||||
| snapshot: RevlogEntry, | snapshot: RevlogEntry, | ||||
| deltas: &[RevlogEntry], | deltas: &[RevlogEntry], | ||||
| ) -> Result<Vec<u8>, RevlogError> { | ) -> Result<Vec<u8>, HgError> { | ||||
| let snapshot = snapshot.data_chunk()?; | let snapshot = snapshot.data_chunk()?; | ||||
| let deltas = deltas | let deltas = deltas | ||||
| .iter() | .iter() | ||||
| .rev() | .rev() | ||||
| .map(RevlogEntry::data_chunk) | .map(RevlogEntry::data_chunk) | ||||
| .collect::<Result<Vec<Cow<'_, [u8]>>, RevlogError>>()?; | .collect::<Result<Vec<_>, _>>()?; | ||||
| let patches: Vec<_> = | let patches: Vec<_> = | ||||
| deltas.iter().map(|d| patch::PatchList::new(d)).collect(); | deltas.iter().map(|d| patch::PatchList::new(d)).collect(); | ||||
| let patch = patch::fold_patch_lists(&patches); | let patch = patch::fold_patch_lists(&patches); | ||||
| Ok(patch.apply(&snapshot)) | Ok(patch.apply(&snapshot)) | ||||
| } | } | ||||
| /// Return the revlog data. | /// Return the revlog data. | ||||
| fn data(&self) -> &[u8] { | fn data(&self) -> &[u8] { | ||||
| match self.data_bytes { | match self.data_bytes { | ||||
| Some(ref data_bytes) => &data_bytes, | Some(ref data_bytes) => &data_bytes, | ||||
| None => panic!( | None => panic!( | ||||
| "forgot to load the data or trying to access inline data" | "forgot to load the data or trying to access inline data" | ||||
| ), | ), | ||||
| } | } | ||||
| } | } | ||||
| /// Get an entry of the revlog. | /// Get an entry of the revlog. | ||||
| fn get_entry(&self, rev: Revision) -> Result<RevlogEntry, RevlogError> { | pub fn get_entry( | ||||
| &self, | |||||
| rev: Revision, | |||||
| ) -> Result<RevlogEntry, RevlogError> { | |||||
| let index_entry = self | let index_entry = self | ||||
| .index | .index | ||||
| .get_entry(rev) | .get_entry(rev) | ||||
| .ok_or(RevlogError::InvalidRevision)?; | .ok_or(RevlogError::InvalidRevision)?; | ||||
| let start = index_entry.offset(); | let start = index_entry.offset(); | ||||
| let end = start + index_entry.compressed_len(); | let end = start + index_entry.compressed_len(); | ||||
| let data = if self.index.is_inline() { | let data = if self.index.is_inline() { | ||||
| self.index.data(start, end) | self.index.data(start, end) | ||||
| Ok(entry) | Ok(entry) | ||||
| } | } | ||||
| /// when resolving internal references within revlog, any errors | /// when resolving internal references within revlog, any errors | ||||
| /// should be reported as corruption, instead of e.g. "invalid revision" | /// should be reported as corruption, instead of e.g. "invalid revision" | ||||
| fn get_entry_internal( | fn get_entry_internal( | ||||
| &self, | &self, | ||||
| rev: Revision, | rev: Revision, | ||||
| ) -> Result<RevlogEntry, RevlogError> { | ) -> Result<RevlogEntry, HgError> { | ||||
| return self.get_entry(rev).map_err(|_| RevlogError::corrupted()); | return self.get_entry(rev).map_err(|_| corrupted()); | ||||
| } | } | ||||
| } | } | ||||
| /// The revlog entry's bytes and the necessary informations to extract | /// The revlog entry's bytes and the necessary informations to extract | ||||
| /// the entry's data. | /// the entry's data. | ||||
| #[derive(Clone)] | #[derive(Clone)] | ||||
| pub struct RevlogEntry<'a> { | pub struct RevlogEntry<'a> { | ||||
| revlog: &'a Revlog, | revlog: &'a Revlog, | ||||
| rev: Revision, | rev: Revision, | ||||
| bytes: &'a [u8], | bytes: &'a [u8], | ||||
| compressed_len: usize, | compressed_len: usize, | ||||
| uncompressed_len: usize, | uncompressed_len: usize, | ||||
| base_rev_or_base_of_delta_chain: Option<Revision>, | base_rev_or_base_of_delta_chain: Option<Revision>, | ||||
| } | } | ||||
| impl<'a> RevlogEntry<'a> { | impl<'a> RevlogEntry<'a> { | ||||
| pub fn revision(&self) -> Revision { | pub fn revision(&self) -> Revision { | ||||
| self.rev | self.rev | ||||
| } | } | ||||
| /// The data for this entry, after resolving deltas if any. | /// The data for this entry, after resolving deltas if any. | ||||
| pub fn data(&self) -> Result<Cow<'a, [u8]>, RevlogError> { | pub fn data(&self) -> Result<Cow<'a, [u8]>, HgError> { | ||||
| let mut entry = self.clone(); | let mut entry = self.clone(); | ||||
| let mut delta_chain = vec![]; | let mut delta_chain = vec![]; | ||||
| // The meaning of `base_rev_or_base_of_delta_chain` depends on | // The meaning of `base_rev_or_base_of_delta_chain` depends on | ||||
| // generaldelta. See the doc on `ENTRY_DELTA_BASE` in | // generaldelta. See the doc on `ENTRY_DELTA_BASE` in | ||||
| // `mercurial/revlogutils/constants.py` and the code in | // `mercurial/revlogutils/constants.py` and the code in | ||||
| // [_chaininfo] and in [index_deltachain]. | // [_chaininfo] and in [index_deltachain]. | ||||
| let uses_generaldelta = self.revlog.index.uses_generaldelta(); | let uses_generaldelta = self.revlog.index.uses_generaldelta(); | ||||
| while let Some(base_rev) = entry.base_rev_or_base_of_delta_chain { | while let Some(base_rev) = entry.base_rev_or_base_of_delta_chain { | ||||
| let base_rev = if uses_generaldelta { | let base_rev = if uses_generaldelta { | ||||
| base_rev | base_rev | ||||
| } else { | } else { | ||||
| entry.rev - 1 | entry.rev - 1 | ||||
| }; | }; | ||||
| delta_chain.push(entry); | delta_chain.push(entry); | ||||
| entry = self.revlog.get_entry_internal(base_rev)?; | entry = self.revlog.get_entry_internal(base_rev)?; | ||||
| } | } | ||||
| // TODO do not look twice in the index | // TODO do not look twice in the index | ||||
| let index_entry = self | let index_entry = self | ||||
| .revlog | .revlog | ||||
| .index | .index | ||||
| .get_entry(self.rev) | .get_entry(self.rev) | ||||
| .ok_or(RevlogError::InvalidRevision)?; | .ok_or_else(corrupted)?; | ||||
| let data = if delta_chain.is_empty() { | let data = if delta_chain.is_empty() { | ||||
| entry.data_chunk()? | entry.data_chunk()? | ||||
| } else { | } else { | ||||
| Revlog::build_data_from_deltas(entry, &delta_chain)?.into() | Revlog::build_data_from_deltas(entry, &delta_chain)?.into() | ||||
| }; | }; | ||||
| if self.revlog.check_hash( | if self.revlog.check_hash( | ||||
| index_entry.p1(), | index_entry.p1(), | ||||
| index_entry.p2(), | index_entry.p2(), | ||||
| index_entry.hash().as_bytes(), | index_entry.hash().as_bytes(), | ||||
| &data, | &data, | ||||
| ) { | ) { | ||||
| Ok(data) | Ok(data) | ||||
| } else { | } else { | ||||
| Err(RevlogError::corrupted()) | Err(corrupted()) | ||||
| } | } | ||||
| } | } | ||||
| /// Extract the data contained in the entry. | /// Extract the data contained in the entry. | ||||
| /// This may be a delta. (See `is_delta`.) | /// This may be a delta. (See `is_delta`.) | ||||
| fn data_chunk(&self) -> Result<Cow<'a, [u8]>, RevlogError> { | fn data_chunk(&self) -> Result<Cow<'a, [u8]>, HgError> { | ||||
| if self.bytes.is_empty() { | if self.bytes.is_empty() { | ||||
| return Ok(Cow::Borrowed(&[])); | return Ok(Cow::Borrowed(&[])); | ||||
| } | } | ||||
| match self.bytes[0] { | match self.bytes[0] { | ||||
| // Revision data is the entirety of the entry, including this | // Revision data is the entirety of the entry, including this | ||||
| // header. | // header. | ||||
| b'\0' => Ok(Cow::Borrowed(self.bytes)), | b'\0' => Ok(Cow::Borrowed(self.bytes)), | ||||
| // Raw revision data follows. | // Raw revision data follows. | ||||
| b'u' => Ok(Cow::Borrowed(&self.bytes[1..])), | b'u' => Ok(Cow::Borrowed(&self.bytes[1..])), | ||||
| // zlib (RFC 1950) data. | // zlib (RFC 1950) data. | ||||
| b'x' => Ok(Cow::Owned(self.uncompressed_zlib_data()?)), | b'x' => Ok(Cow::Owned(self.uncompressed_zlib_data()?)), | ||||
| // zstd data. | // zstd data. | ||||
| b'\x28' => Ok(Cow::Owned(self.uncompressed_zstd_data()?)), | b'\x28' => Ok(Cow::Owned(self.uncompressed_zstd_data()?)), | ||||
| // A proper new format should have had a repo/store requirement. | // A proper new format should have had a repo/store requirement. | ||||
| _format_type => Err(RevlogError::corrupted()), | _format_type => Err(corrupted()), | ||||
| } | } | ||||
| } | } | ||||
| fn uncompressed_zlib_data(&self) -> Result<Vec<u8>, RevlogError> { | fn uncompressed_zlib_data(&self) -> Result<Vec<u8>, HgError> { | ||||
| let mut decoder = ZlibDecoder::new(self.bytes); | let mut decoder = ZlibDecoder::new(self.bytes); | ||||
| if self.is_delta() { | if self.is_delta() { | ||||
| let mut buf = Vec::with_capacity(self.compressed_len); | let mut buf = Vec::with_capacity(self.compressed_len); | ||||
| decoder | decoder.read_to_end(&mut buf).map_err(|_| corrupted())?; | ||||
| .read_to_end(&mut buf) | |||||
| .map_err(|_| RevlogError::corrupted())?; | |||||
| Ok(buf) | Ok(buf) | ||||
| } else { | } else { | ||||
| let mut buf = vec![0; self.uncompressed_len]; | let mut buf = vec![0; self.uncompressed_len]; | ||||
| decoder | decoder.read_exact(&mut buf).map_err(|_| corrupted())?; | ||||
| .read_exact(&mut buf) | |||||
| .map_err(|_| RevlogError::corrupted())?; | |||||
| Ok(buf) | Ok(buf) | ||||
| } | } | ||||
| } | } | ||||
| fn uncompressed_zstd_data(&self) -> Result<Vec<u8>, RevlogError> { | fn uncompressed_zstd_data(&self) -> Result<Vec<u8>, HgError> { | ||||
| if self.is_delta() { | if self.is_delta() { | ||||
| let mut buf = Vec::with_capacity(self.compressed_len); | let mut buf = Vec::with_capacity(self.compressed_len); | ||||
| zstd::stream::copy_decode(self.bytes, &mut buf) | zstd::stream::copy_decode(self.bytes, &mut buf) | ||||
| .map_err(|_| RevlogError::corrupted())?; | .map_err(|_| corrupted())?; | ||||
| Ok(buf) | Ok(buf) | ||||
| } else { | } else { | ||||
| let mut buf = vec![0; self.uncompressed_len]; | let mut buf = vec![0; self.uncompressed_len]; | ||||
| let len = zstd::block::decompress_to_buffer(self.bytes, &mut buf) | let len = zstd::block::decompress_to_buffer(self.bytes, &mut buf) | ||||
| .map_err(|_| RevlogError::corrupted())?; | .map_err(|_| corrupted())?; | ||||
| if len != self.uncompressed_len { | if len != self.uncompressed_len { | ||||
| Err(RevlogError::corrupted()) | Err(corrupted()) | ||||
| } else { | } else { | ||||
| Ok(buf) | Ok(buf) | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| /// Tell if the entry is a snapshot or a delta | /// Tell if the entry is a snapshot or a delta | ||||
| /// (influences on decompression). | /// (influences on decompression). | ||||
| let entry = manifest | let entry = manifest | ||||
| .find_by_path(hg_path)? | .find_by_path(hg_path)? | ||||
| .expect("ambgious file not in p1"); | .expect("ambgious file not in p1"); | ||||
| if entry.flags != fs_flags { | if entry.flags != fs_flags { | ||||
| return Ok(true); | return Ok(true); | ||||
| } | } | ||||
| let filelog = repo.filelog(hg_path)?; | let filelog = repo.filelog(hg_path)?; | ||||
| let fs_len = fs_metadata.len(); | let fs_len = fs_metadata.len(); | ||||
| let filelog_entry = | |||||
| filelog.entry_for_node(entry.node_id()?).map_err(|_| { | |||||
| HgError::corrupted("filelog missing node from manifest") | |||||
| })?; | |||||
| // TODO: check `fs_len` here like below, but based on | // TODO: check `fs_len` here like below, but based on | ||||
| // `RevlogEntry::uncompressed_len` without decompressing the full filelog | // `RevlogEntry::uncompressed_len` without decompressing the full filelog | ||||
| // contents where possible. This is only valid if the revlog data does not | // contents where possible. This is only valid if the revlog data does not | ||||
| // contain metadata. See how Python’s `revlog.rawsize` calls | // contain metadata. See how Python’s `revlog.rawsize` calls | ||||
| // `storageutil.filerevisioncopied`. | // `storageutil.filerevisioncopied`. | ||||
| // (Maybe also check for content-modifying flags? See `revlog.size`.) | // (Maybe also check for content-modifying flags? See `revlog.size`.) | ||||
| let filelog_entry = | let filelog_data = filelog_entry.data()?; | ||||
| filelog.data_for_node(entry.node_id()?).map_err(|_| { | let contents_in_p1 = filelog_data.file_data()?; | ||||
| HgError::corrupted("filelog missing node from manifest") | |||||
| })?; | |||||
| let contents_in_p1 = filelog_entry.file_data()?; | |||||
| if contents_in_p1.len() as u64 != fs_len { | if contents_in_p1.len() as u64 != fs_len { | ||||
| // No need to read the file contents: | // No need to read the file contents: | ||||
| // it cannot be equal if it has a different length. | // it cannot be equal if it has a different length. | ||||
| return Ok(true); | return Ok(true); | ||||
| } | } | ||||
| let fs_contents = if is_symlink { | let fs_contents = if is_symlink { | ||||
| get_bytes_from_os_string(vfs.read_link(fs_path)?.into_os_string()) | get_bytes_from_os_string(vfs.read_link(fs_path)?.into_os_string()) | ||||
| } else { | } else { | ||||
| vfs.read(fs_path)? | vfs.read(fs_path)? | ||||
| }; | }; | ||||
| Ok(contents_in_p1 != &*fs_contents) | Ok(contents_in_p1 != &*fs_contents) | ||||
| } | } | ||||