Skip to content

Fix issues with path resolving and listing files #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
volodymyrpoli wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Fix issues with path resolving and listing files #23

Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
eb695ec
Fix paths issues (double slashes, end slashes)
volodymyrpoli Dec 17, 2020
deda598
Fix recursive listing of files
volodymyrpoli Dec 17, 2020
fef341a
Fix overwrite parameter conversion
volodymyrpoli Dec 17, 2020
35837f2
Replace maxAmountOfFiles with maxDepth, as it is a more expected result
volodymyrpoli Dec 21, 2020
86cfad8
Update package name and version
volodymyrpoli Dec 21, 2020
f5b9b71
[BUGFIX] DP-460
daniilly Jan 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 49 additions & 13 deletions src/org/apache/hadoop/WebHDFS.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public function create(

$options = array(
'op' => 'CREATE',
'overwrite' => $overwrite,
'overwrite' => $overwrite ? 'true' : 'false',
'blocksize' => $blocksize,
'replication' => $replication,
'permission' => $permission,
Expand All @@ -79,7 +79,7 @@ public function createWithData(
) {
$options = array(
'op' => 'CREATE',
'overwrite' => $overwrite,
'overwrite' => $overwrite ? 'true' : 'false',
'blocksize' => $blockSize,
'replication' => $replication,
'permission' => $permission,
Expand Down Expand Up @@ -200,7 +200,15 @@ private function _listStatus($path, $cleanLastRequest = false)
}
}

public function listFiles($path, $recursive = false, $includeFileMetaData = false, $maxAmountOfFiles = false)
/**
* @param string $path
* @param bool $recursive
* @param bool $includeFileMetaData
* @param int $maxAmountOfFiles
* @return array
* @throws WebHDFS_Exception
*/
public function listFiles($path, $recursive = false, $includeFileMetaData = false, $maxAmountOfFiles = 0)
{
$result = array();
$listStatusResult = $this->_listStatus($path);
Expand All @@ -211,21 +219,27 @@ public function listFiles($path, $recursive = false, $includeFileMetaData = fals
switch ($fileEntity->type) {
case 'DIRECTORY':
if ($recursive === true) {
$result = array_merge($result,
$this->listFiles($path.$fileEntity->pathSuffix.'/', true, $includeFileMetaData,
$maxAmountOfFiles - sizeof($result)));
$result = array_merge(
$result,
$this->listFiles(
$this->concatPath([$path, $fileEntity->pathSuffix]),
true,
$includeFileMetaData,
$maxAmountOfFiles - sizeof($result)
)
);
}
break;
default:
if ($includeFileMetaData === true) {
$fileEntity->path = $path.$fileEntity->pathSuffix;
$fileEntity->path = $this->concatPath([$path, $fileEntity->pathSuffix]);
$result[] = $fileEntity;
} else {
$result[] = $path.$fileEntity->pathSuffix;
$result[] = $this->concatPath([$path, $fileEntity->pathSuffix]);
}
}
// recursion will be interrupted since we subtract the amount of the current result set from the maxAmountOfFiles amount with calling the next recursion
if (sizeof($result) >= $maxAmountOfFiles) {
if ($maxAmountOfFiles !== 0 && sizeof($result) >= $maxAmountOfFiles) {
break;
}
}
Expand All @@ -247,15 +261,19 @@ public function listDirectories($path, $recursive = false, $includeFileMetaData
switch ($fileEntity->type) {
case 'DIRECTORY':
if ($includeFileMetaData === true) {
$fileEntity->path = $path.$fileEntity->pathSuffix;
$fileEntity->path = $this->concatPath([$path, $fileEntity->pathSuffix]);
$result[] = $fileEntity;
} else {
$result[] = $path.$fileEntity->pathSuffix;
$result[] = $this->concatPath([$path, $fileEntity->pathSuffix]);
}
if ($recursive === true) {
$result = array_merge($result,
$this->listDirectories($path.$fileEntity->pathSuffix.'/', $recursive,
$includeFileMetaData));
$this->listDirectories(
$this->concatPath([$path, $fileEntity->pathSuffix]),
$recursive,
$includeFileMetaData
)
);
}
break;
}
Expand Down Expand Up @@ -445,4 +463,22 @@ private function getResponseErrorException($responseData)

return new WebHDFS_Exception($exceptionMessage, $exceptionCode);
}

private function concatPath(array $paths) {
$result = '';
foreach ($paths as $path) {
if (!$result || preg_match('/.+\/$/', $result)) {
$result .= $path;
continue;
}

$result .= '/' . $path;
}
return $this->removeMultiSlashFromPath($result);
}

private function removeMultiSlashFromPath($path) {
return preg_replace('/(\/)\/+/', '$1', $path);
}

}