diff --git a/Changelog.md b/Changelog.md index c32ef83f5b..6e94760b86 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,7 +1,8 @@ # FOSSA CLI Changelog -## Unreleased +## 3.16.8 +- Poetry: Support PEP 621 `[project].dependencies` for Poetry 2.x projects. Production dependencies declared in the standard `[project]` section are now correctly detected alongside legacy `[tool.poetry.dependencies]`. ([#1683](https://github.com/fossas/fossa-cli/pull/1683)) - UV: Fix fatal parse error on uv.lock files containing editable/workspace packages with dynamic versions ([#1682](https://github.com/fossas/fossa-cli/pull/1682)) - Gradle: Add additional development and test configurations for common plugins ([#1684](https://github.com/fossas/fossa-cli/pull/1684)) diff --git a/src/Strategy/Python/PDM/PdmLock.hs b/src/Strategy/Python/PDM/PdmLock.hs index a34c15a651..208c888a41 100644 --- a/src/Strategy/Python/PDM/PdmLock.hs +++ b/src/Strategy/Python/PDM/PdmLock.hs @@ -13,7 +13,7 @@ import Data.Text (Text) import DepTypes (DepEnvironment (EnvDevelopment, EnvProduction), DepType (GitType, PipType, URLType, UnresolvedPathType), Dependency (..), VerConstraint (..), hydrateDepEnvs) import Effect.Grapher (deep, direct, edge, evalGrapher, run) import Graphing (Graphing, gmap) -import Strategy.Python.Util (Req (..)) +import Strategy.Python.Util (Req (..), reqName) import Toml.Schema qualified -- | Represents pdm lock file. @@ -131,10 +131,6 @@ toDependency prodReqs devReqs pkg = isUrlReq (UrlReq{}) = True isUrlReq _ = False -reqName :: Req -> Text -reqName (NameReq rname _ _ _) = rname -reqName (UrlReq rname _ _ _) = rname - buildGraph :: [Req] -> [Req] -> PdmLock -> Graphing Dependency buildGraph prodReqs devReqs pdmLock = hydrateDepEnvs $ gmap (toDependency prodReqs devReqs) $ diff --git a/src/Strategy/Python/Poetry/Common.hs b/src/Strategy/Python/Poetry/Common.hs index 7aa23ad55d..5be9558832 100644 --- a/src/Strategy/Python/Poetry/Common.hs +++ b/src/Strategy/Python/Poetry/Common.hs @@ -30,6 +30,7 @@ import Strategy.Python.Poetry.PyProject ( PoetryDependency (..), PyProject (..), PyProjectBuildSystem (..), + PyProjectMetadata (..), PyProjectPoetry (..), PyProjectPoetryDetailedVersionDependency (..), PyProjectPoetryGitDependency (..), @@ -41,6 +42,7 @@ import Strategy.Python.Poetry.PyProject ( allPoetryNonProductionDeps, toDependencyVersion, ) +import Strategy.Python.Util (reqToDependency) -- | Gets build backend of pyproject. getPoetryBuildBackend :: PyProject -> Maybe Text @@ -126,13 +128,27 @@ pyProjectDeps project = filter notNamedPython $ map snd allDeps Just (PyProjectTool{pyprojectPoetry}) -> maybe Map.empty dependencies pyprojectPoetry _ -> mempty + -- PEP 621 [project].dependencies converted to Dependency with EnvProduction + pep621ProdDeps :: Map Text Dependency + pep621ProdDeps = case pyprojectProject project of + Just (PyProjectMetadata{pyprojectDependencies = Just reqs}) -> + Map.fromList $ map toNamedProdDep reqs + _ -> mempty + where + toNamedProdDep req = + let dep = (reqToDependency req){dependencyEnvironments = Set.singleton EnvProduction} + in (dependencyName dep, dep) + toDependency :: [DepEnvironment] -> Map Text PoetryDependency -> Map Text Dependency toDependency depEnvs = Map.mapWithKey $ poetrytoDependency depEnvs allDeps :: [(Text, Dependency)] allDeps = Map.toList prodDeps ++ Map.toList devDeps where - prodDeps = toDependency [EnvProduction] supportedProdDeps + -- Poetry-style prod deps take precedence; PEP 621 fills in anything missing. + -- [tool.poetry.dependencies] can carry richer metadata (explicit source, git refs, etc.) + -- that PEP 621's PEP 508 strings can't express, so it's strictly more informative. + prodDeps = toDependency [EnvProduction] supportedProdDeps `Map.union` pep621ProdDeps devDeps = toDependency [EnvDevelopment] supportedDevDeps -- | Gets Dependency from `PoetryDependency` and it's `DepEnvironment`. diff --git a/src/Strategy/Python/Poetry/PyProject.hs b/src/Strategy/Python/Poetry/PyProject.hs index 29bb2e5aff..ee66f0c18c 100644 --- a/src/Strategy/Python/Poetry/PyProject.hs +++ b/src/Strategy/Python/Poetry/PyProject.hs @@ -21,9 +21,9 @@ module Strategy.Python.Poetry.PyProject ( ) where import Control.Monad.Combinators.Expr (Operator (..), makeExprParser) -import Data.Foldable (asum) +import Data.Foldable (asum, foldl') import Data.Functor (void) -import Data.Map (Map, unions) +import Data.Map (Map, insert, union, unions) import Data.Maybe (fromMaybe) import Data.String.Conversion (toString, toText) import Data.Text (Text) @@ -41,7 +41,7 @@ import DepTypes ( COr ), ) -import Strategy.Python.Util (Req) +import Strategy.Python.Util (Req (..), reqName) import Text.Megaparsec ( Parsec, empty, @@ -188,12 +188,28 @@ instance Toml.Schema.FromValue PyProjectPoetryGroupDependencies where PyProjectPoetryGroupDependencies <$> Toml.Schema.pickKey [Toml.Schema.Key "dependencies" Toml.Schema.fromValue, Toml.Schema.Else (pure mempty)] +-- | Returns all production dependencies from a pyproject.toml. +-- Merges legacy [tool.poetry.dependencies] with PEP 621 [project].dependencies. +-- Poetry-style entries take precedence when a dep appears in both sections. allPoetryProductionDeps :: PyProject -> Map Text PoetryDependency -allPoetryProductionDeps project = case pyprojectTool project of - Just (PyProjectTool{pyprojectPoetry}) -> case pyprojectPoetry of - Just (PyProjectPoetry{dependencies}) -> dependencies - _ -> mempty - _ -> mempty +allPoetryProductionDeps project = poetryDeps `union` pep621Deps + where + -- Legacy [tool.poetry.dependencies] — takes precedence in union + poetryDeps = case pyprojectTool project of + Just (PyProjectTool{pyprojectPoetry}) -> case pyprojectPoetry of + Just (PyProjectPoetry{dependencies}) -> dependencies + _ -> mempty + _ -> mempty + + -- PEP 621 [project].dependencies — fallback for deps not in poetryDeps. + -- We use PoetryTextVersion "*" as a placeholder since the actual version + -- comes from the lock file. The key purpose is to register the package name + -- so it's recognized as a production dependency. + pep621Deps :: Map Text PoetryDependency + pep621Deps = case pyprojectProject project of + Just (PyProjectMetadata{pyprojectDependencies = Just reqs}) -> + foldl' (\acc req -> insert (reqName req) (PoetryTextVersion "*") acc) mempty reqs + _ -> mempty allPoetryNonProductionDeps :: PyProject -> Map Text PoetryDependency allPoetryNonProductionDeps project = unions [olderPoetryDevDeps, optionalDeps] diff --git a/src/Strategy/Python/Util.hs b/src/Strategy/Python/Util.hs index 3f271c3faa..4cf1c21d92 100644 --- a/src/Strategy/Python/Util.hs +++ b/src/Strategy/Python/Util.hs @@ -6,6 +6,7 @@ module Strategy.Python.Util ( MarkerOp (..), Operator (..), Req (..), + reqName, requirementParser, reqToDependency, toConstraint, @@ -36,15 +37,11 @@ pkgToReq :: PythonPackage -> Req pkgToReq p = NameReq (pkgName p) Nothing (Just [Version OpEq (pkgVersion p)]) Nothing -depName :: Req -> Text -depName (NameReq nm _ _ _) = nm -depName (UrlReq nm _ _ _) = nm - reqToDependency :: Req -> Dependency reqToDependency req = Dependency { dependencyType = PipType - , dependencyName = depName req + , dependencyName = reqName req , dependencyVersion = depVersion req , dependencyLocations = [] , dependencyEnvironments = mempty @@ -94,7 +91,7 @@ buildGraph maybePackages reqs = do Nothing -> pure () where findParent :: Text -> Maybe Req - findParent packageName = find (\r -> Text.toLower (depName r) == Text.toLower (packageName)) reqs + findParent packageName = find (\r -> Text.toLower (reqName r) == Text.toLower (packageName)) reqs addChildren :: (Has (Grapher Req) sig m) => Req -> PythonPackage -> m () addChildren parent pkg = do @@ -174,6 +171,10 @@ data Req | UrlReq Text (Maybe [Text]) URI.URI (Maybe Marker) -- name, extras, ... deriving (Eq, Ord, Show) +reqName :: Req -> Text +reqName (NameReq name _ _ _) = name +reqName (UrlReq name _ _ _) = name + instance Toml.Schema.FromValue Req where fromValue v = do value <- Toml.Schema.fromValue v diff --git a/test/Python/Poetry/CommonSpec.hs b/test/Python/Poetry/CommonSpec.hs index 01d80a330a..79dec694d8 100644 --- a/test/Python/Poetry/CommonSpec.hs +++ b/test/Python/Poetry/CommonSpec.hs @@ -25,6 +25,7 @@ import Strategy.Python.Poetry.PyProject ( PyProjectPoetryPathDependency (..), PyProjectPoetryUrlDependency (..), PyProjectTool (..), + allPoetryProductionDeps, ) import Test.Hspec ( Spec, @@ -159,6 +160,8 @@ spec :: Spec spec = do nominalContents <- runIO (TIO.readFile "test/Python/Poetry/testdata/pyproject1.toml") emptyContents <- runIO (TIO.readFile "test/Python/Poetry/testdata/pyproject2.toml") + pep621Contents <- runIO (TIO.readFile "test/Python/Poetry/testdata/pep621/pyproject.toml") + pep621MixedContents <- runIO (TIO.readFile "test/Python/Poetry/testdata/pep621-mixed/pyproject.toml") describe "toCanonicalName" $ do it "should convert text to lowercase" $ @@ -166,10 +169,51 @@ spec = do it "should replace underscore (_) to hyphens (-)" $ toCanonicalName "my_oh_so_great_pkg" `shouldBe` "my-oh-so-great-pkg" - describe "getDependencies" $ + describe "getDependencies" $ do it "should get all dependencies" $ pyProjectDeps expectedPyProject `shouldMatchList` expectedDeps + it "should get PEP 621 [project].dependencies as production deps" $ do + case decodeEither pep621Contents of + Left e -> fail $ "Failed to parse pyproject.toml: " <> show e + Right pyProject -> do + let deps = pyProjectDeps pyProject + let prodDeps = filter (\d -> Set.singleton EnvProduction == dependencyEnvironments d) deps + let devDeps = filter (\d -> Set.singleton EnvDevelopment == dependencyEnvironments d) deps + map dependencyName prodDeps `shouldMatchList` ["requests", "flask"] + map dependencyName devDeps `shouldMatchList` ["pytest"] + + it "should prefer [tool.poetry.dependencies] over [project].dependencies for same package" $ do + case decodeEither pep621MixedContents of + Left e -> fail $ "Failed to parse pyproject.toml: " <> show e + Right pyProject -> do + let deps = pyProjectDeps pyProject + let prodDeps = filter (\d -> Set.singleton EnvProduction == dependencyEnvironments d) deps + -- Both requests and flask should be production deps + map dependencyName prodDeps `shouldMatchList` ["requests", "flask"] + + describe "allPoetryProductionDeps" $ do + it "should include PEP 621 [project].dependencies" $ do + case decodeEither pep621Contents of + Left e -> fail $ "Failed to parse pyproject.toml: " <> show e + Right pyProject -> do + let prodDeps = allPoetryProductionDeps pyProject + Map.member "requests" prodDeps `shouldBe` True + Map.member "flask" prodDeps `shouldBe` True + + it "should prefer [tool.poetry.dependencies] entries over PEP 621 entries" $ do + case decodeEither pep621MixedContents of + Left e -> fail $ "Failed to parse pyproject.toml: " <> show e + Right pyProject -> do + let prodDeps = allPoetryProductionDeps pyProject + -- requests is in both sections; poetry entry (DetailedVersion) should win + case Map.lookup "requests" prodDeps of + Just (PyProjectPoetryDetailedVersionDependencySpec _) -> pure () + Just other -> fail $ "Expected DetailedVersionDependencySpec for requests, got: " <> show other + Nothing -> fail "requests not found in production deps" + -- flask is only in [project].dependencies + Map.member "flask" prodDeps `shouldBe` True + describe "supportedPyProjectDep" $ it "should return false when dependency is sourced from local path" $ supportedPyProjectDep notSupportedPyProjectDependency `shouldBe` False diff --git a/test/Python/Poetry/testdata/pep621-mixed/pyproject.toml b/test/Python/Poetry/testdata/pep621-mixed/pyproject.toml new file mode 100644 index 0000000000..7488cf747a --- /dev/null +++ b/test/Python/Poetry/testdata/pep621-mixed/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "my-mixed-project" +version = "1.0.0" +dependencies = [ + "requests>=2.28.0", + "flask>=2.0", +] + +[tool.poetry.dependencies] +# Poetry-style entry for requests with explicit source — should take precedence +requests = {version = ">=2.28.0"} + +[tool.poetry.group.dev.dependencies] +pytest = "^6.0" + +[build-system] +requires = ["poetry-core>=2.0"] +build-backend = "poetry.core.masonry.api" diff --git a/test/Python/Poetry/testdata/pep621/pyproject.toml b/test/Python/Poetry/testdata/pep621/pyproject.toml new file mode 100644 index 0000000000..f68f9ea3f0 --- /dev/null +++ b/test/Python/Poetry/testdata/pep621/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "my-pep621-project" +version = "1.0.0" +dependencies = [ + "requests>=2.28.0", + "flask>=2.0", +] + +[tool.poetry.group.dev.dependencies] +pytest = "^6.0" + +[build-system] +requires = ["poetry-core>=2.0"] +build-backend = "poetry.core.masonry.api"